diff --git a/archives/filecount_cleanup_745cac8f/hydra_outputs_2026_05_to_07.zip b/archives/filecount_cleanup_745cac8f/hydra_outputs_2026_05_to_07.zip new file mode 100644 index 0000000000000000000000000000000000000000..1e45ceee122fdc21942a73029c6f591d98bf5ade --- /dev/null +++ b/archives/filecount_cleanup_745cac8f/hydra_outputs_2026_05_to_07.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51235adbd72b93a3d6226821e07053c351faf5633a9700c201285c7a62f0713d +size 1450750 diff --git a/archives/filecount_cleanup_745cac8f/wandb_segment_csv_2026_06_to_07.zip b/archives/filecount_cleanup_745cac8f/wandb_segment_csv_2026_06_to_07.zip new file mode 100644 index 0000000000000000000000000000000000000000..f1242a8060c1c75a5029a3815a9dbfeb90a9778c --- /dev/null +++ b/archives/filecount_cleanup_745cac8f/wandb_segment_csv_2026_06_to_07.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c24792417027a4fc7f88ff62f3e09892dd3b8b7aa7885087634f82bb94b1fc4 +size 546687 diff --git a/outputs/2026-05-31/16-33-10/.hydra/config.yaml b/outputs/2026-05-31/16-33-10/.hydra/config.yaml deleted file mode 100644 index 37f20840bdb620aecfcd1d64574488562bcf3127..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-10/.hydra/config.yaml +++ /dev/null @@ -1,235 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 8 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 700 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: DeMemWM - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-05-31/16-33-10/.hydra/hydra.yaml b/outputs/2026-05-31/16-33-10/.hydra/hydra.yaml deleted file mode 100644 index fd70e7cc2cf00438022c75d6c5c11debb1b0b5ff..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-10/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=700 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=8 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=700,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=8,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-05-31/16-33-10 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-05-31/16-33-10/.hydra/overrides.yaml b/outputs/2026-05-31/16-33-10/.hydra/overrides.yaml deleted file mode 100644 index 47fd8dfd4320154e59dcf5d378b89f43446db9b2..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-10/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=700 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=8 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-05-31/16-33-10/main.log b/outputs/2026-05-31/16-33-10/main.log deleted file mode 100644 index 4988357ba5cf33d78d57e07abfaf6c1774e7d009..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-10/main.log +++ /dev/null @@ -1,38 +0,0 @@ -[2026-05-31 16:34:05,141][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-05-31 16:34:13,901][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-05-31 16:34:18,601][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-05-31 16:34:18,897][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-05-31 16:34:19,141][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-05-31 16:34:19,288][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 16:33:00,182][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,182][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-03 16:33:00,186][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-03 16:33:00,190][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:01,747][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-03 16:33:01,748][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:01,748][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-03 16:33:01,749][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-05-31/16-33-11/.hydra/config.yaml b/outputs/2026-05-31/16-33-11/.hydra/config.yaml deleted file mode 100644 index 37f20840bdb620aecfcd1d64574488562bcf3127..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-11/.hydra/config.yaml +++ /dev/null @@ -1,235 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 8 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 700 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: DeMemWM - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-05-31/16-33-11/.hydra/hydra.yaml b/outputs/2026-05-31/16-33-11/.hydra/hydra.yaml deleted file mode 100644 index 1e7e791218d6ed726b9807159f5e84e551825612..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-11/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=700 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=8 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=700,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=8,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-05-31/16-33-11 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-05-31/16-33-11/.hydra/overrides.yaml b/outputs/2026-05-31/16-33-11/.hydra/overrides.yaml deleted file mode 100644 index 47fd8dfd4320154e59dcf5d378b89f43446db9b2..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-11/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=700 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=8 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-05-31/16-33-11/main.log b/outputs/2026-05-31/16-33-11/main.log deleted file mode 100644 index 272f8f57cb3a695f81cdb0a6d94b95a61cb322f2..0000000000000000000000000000000000000000 --- a/outputs/2026-05-31/16-33-11/main.log +++ /dev/null @@ -1,39 +0,0 @@ -[2026-05-31 16:33:58,507][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-05-31 16:33:58,787][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-05-31 16:34:01,148][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-05-31 16:34:02,180][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-05-31 16:34:05,142][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-05-31 16:34:14,613][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-05-31 16:34:20,380][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-05-31 16:34:34,882][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:34,883][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-05-31 16:34:36,691][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 199 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -911 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,573.841 Total estimated model params size (MB) -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-05-31 16:34:39,689][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 16:33:00,183][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-03 16:33:00,184][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-03 16:33:00,189][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-03 16:33:01,749][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-03 16:33:01,750][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-01/15-41-01/.hydra/config.yaml b/outputs/2026-06-01/15-41-01/.hydra/config.yaml deleted file mode 100644 index 524d4d518b4f396f3129bef0c51692c254595dc9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-01/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-01/15-41-01/.hydra/hydra.yaml b/outputs/2026-06-01/15-41-01/.hydra/hydra.yaml deleted file mode 100644 index 241213c048d14694dff0605716740d2924303b49..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-01/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors - - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft - - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-01 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-01/15-41-01/.hydra/overrides.yaml b/outputs/2026-06-01/15-41-01/.hydra/overrides.yaml deleted file mode 100644 index 5ad2fae2e9cad8bfdc384b11f419b198a35e608d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-01/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft -- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-01/15-41-01/main.log b/outputs/2026-06-01/15-41-01/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-01/15-41-06/.hydra/config.yaml b/outputs/2026-06-01/15-41-06/.hydra/config.yaml deleted file mode 100644 index 524d4d518b4f396f3129bef0c51692c254595dc9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-06/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-01/15-41-06/.hydra/hydra.yaml b/outputs/2026-06-01/15-41-06/.hydra/hydra.yaml deleted file mode 100644 index 4326c507342c10f1a725d2d6946937ed7382db0e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-06/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors - - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft - - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-06 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-01/15-41-06/.hydra/overrides.yaml b/outputs/2026-06-01/15-41-06/.hydra/overrides.yaml deleted file mode 100644 index 5ad2fae2e9cad8bfdc384b11f419b198a35e608d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-06/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft -- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-01/15-41-06/main.log b/outputs/2026-06-01/15-41-06/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-01/15-41-07/.hydra/config.yaml b/outputs/2026-06-01/15-41-07/.hydra/config.yaml deleted file mode 100644 index 524d4d518b4f396f3129bef0c51692c254595dc9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-07/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-01/15-41-07/.hydra/hydra.yaml b/outputs/2026-06-01/15-41-07/.hydra/hydra.yaml deleted file mode 100644 index 236d86387dd98ce428f76a20afaea78060333986..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-07/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors - - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft - - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-07 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-01/15-41-07/.hydra/overrides.yaml b/outputs/2026-06-01/15-41-07/.hydra/overrides.yaml deleted file mode 100644 index 5ad2fae2e9cad8bfdc384b11f419b198a35e608d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-07/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft -- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-01/15-41-07/main.log b/outputs/2026-06-01/15-41-07/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-01/15-41-08/.hydra/config.yaml b/outputs/2026-06-01/15-41-08/.hydra/config.yaml deleted file mode 100644 index 524d4d518b4f396f3129bef0c51692c254595dc9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-08/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /share_1/users/bonan_ding/worldmem_data/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -vae_path: /share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-01/15-41-08/.hydra/hydra.yaml b/outputs/2026-06-01/15-41-08/.hydra/hydra.yaml deleted file mode 100644 index 947a3c4e9eecddc25e23cbe9b9ed0a321639fcff..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-08/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors - - +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft - - dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features,dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-01/15-41-08 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-01/15-41-08/.hydra/overrides.yaml b/outputs/2026-06-01/15-41-08/.hydra/overrides.yaml deleted file mode 100644 index 5ad2fae2e9cad8bfdc384b11f419b198a35e608d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-08/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/oasis500m.safetensors -- +vae_path=/share_1/users/bonan_ding/WorldMem/open-oasis/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/share_1/users/bonan_ding/worldmem_data/minecraft -- dataset.precomputed_feature_dir=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-01/15-41-08/main.log b/outputs/2026-06-01/15-41-08/main.log deleted file mode 100644 index 0514aa6f3467861d38e1830185e8ef48a7b12d9e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-01/15-41-08/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-01 15:41:49,208][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-01 15:41:49,334][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs diff --git a/outputs/2026-06-03/20-22-54/.hydra/config.yaml b/outputs/2026-06-03/20-22-54/.hydra/config.yaml deleted file mode 100644 index bc967412e28848c9107a72c04ac40cbb8f36791b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-54/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-03/20-22-54/.hydra/hydra.yaml b/outputs/2026-06-03/20-22-54/.hydra/hydra.yaml deleted file mode 100644 index c8967584d5c9855c45197e0d05b0360aec9c8471..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-54/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-54 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-03/20-22-54/.hydra/overrides.yaml b/outputs/2026-06-03/20-22-54/.hydra/overrides.yaml deleted file mode 100644 index 1de347908bc11f34050a2011e470bf96dc9d5be4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-54/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-03/20-22-54/main.log b/outputs/2026-06-03/20-22-54/main.log deleted file mode 100644 index f755f313681872bd09cfbac3089458f620a4c049..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-54/main.log +++ /dev/null @@ -1,47 +0,0 @@ -[2026-06-03 20:23:36,031][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-03 20:23:36,800][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-03 20:23:38,339][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-03 20:23:39,236][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-03 20:23:42,001][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-03 20:23:42,005][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-03 20:23:42,015][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-03 20:23:42,026][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-03 20:23:48,581][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:05,745][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 631 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -632 M Trainable params -231 M Non-trainable params -864 M Total params -3,456.995 Total estimated model params size (MB) -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-04 06:21:13,814][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-04 06:21:13,814][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-03/20-22-55/.hydra/config.yaml b/outputs/2026-06-03/20-22-55/.hydra/config.yaml deleted file mode 100644 index bc967412e28848c9107a72c04ac40cbb8f36791b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-55/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-03/20-22-55/.hydra/hydra.yaml b/outputs/2026-06-03/20-22-55/.hydra/hydra.yaml deleted file mode 100644 index 6a19691cd6bafafb4a5b56a2ce0629703bebfb39..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-55/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-55 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-03/20-22-55/.hydra/overrides.yaml b/outputs/2026-06-03/20-22-55/.hydra/overrides.yaml deleted file mode 100644 index 1de347908bc11f34050a2011e470bf96dc9d5be4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-55/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-03/20-22-55/main.log b/outputs/2026-06-03/20-22-55/main.log deleted file mode 100644 index 3f44cc084f0c750896c8374c9eb97446321660f2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-55/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-03 20:23:41,996][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-03 20:23:42,011][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-03/20-22-56/.hydra/config.yaml b/outputs/2026-06-03/20-22-56/.hydra/config.yaml deleted file mode 100644 index bc967412e28848c9107a72c04ac40cbb8f36791b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-56/.hydra/config.yaml +++ /dev/null @@ -1,234 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 2 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - debug_force_all_streams: false - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - stage_policy: - noise_bucket_logging: true - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-03/20-22-56/.hydra/hydra.yaml b/outputs/2026-06-03/20-22-56/.hydra/hydra.yaml deleted file mode 100644 index b102173e31cb149ba6b368db51c49d75e65a1abf..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-56/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_force_all_streams=false - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.stage_policy.noise_bucket_logging=true - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=2 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_force_all_streams=false,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.stage_policy.noise_bucket_logging=true,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=2,experiment.validation.val_every_n_step=2000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-03/20-22-56 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-03/20-22-56/.hydra/overrides.yaml b/outputs/2026-06-03/20-22-56/.hydra/overrides.yaml deleted file mode 100644 index 1de347908bc11f34050a2011e470bf96dc9d5be4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-56/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_force_all_streams=false -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.stage_policy.noise_bucket_logging=true -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=2 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-03/20-22-56/main.log b/outputs/2026-06-03/20-22-56/main.log deleted file mode 100644 index dd2a8c604e8450b593daa9824b78acc23b90952a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-03/20-22-56/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-03 20:23:42,010][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-03 20:23:42,017][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:03,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-03 20:24:08,742][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:21:13,812][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-04 06:21:13,813][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-04/06-22-27/.hydra/config.yaml b/outputs/2026-06-04/06-22-27/.hydra/config.yaml deleted file mode 100644 index 3640bb3cbb3cb04f1b3f8269c7c813f70d37cc96..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-27/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-22-27/.hydra/hydra.yaml b/outputs/2026-06-04/06-22-27/.hydra/hydra.yaml deleted file mode 100644 index 9728d6e3da029ca14b33ba70bc101a4014d8c851..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-27/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-22-27 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-22-27/.hydra/overrides.yaml b/outputs/2026-06-04/06-22-27/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-27/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-22-27/main.log b/outputs/2026-06-04/06-22-27/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-04/06-22-28/.hydra/config.yaml b/outputs/2026-06-04/06-22-28/.hydra/config.yaml deleted file mode 100644 index 3640bb3cbb3cb04f1b3f8269c7c813f70d37cc96..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-28/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-22-28/.hydra/hydra.yaml b/outputs/2026-06-04/06-22-28/.hydra/hydra.yaml deleted file mode 100644 index d005c001f5cdedc82042c05a287d29358d89749a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-28/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-22-28 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-22-28/.hydra/overrides.yaml b/outputs/2026-06-04/06-22-28/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-22-28/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-22-28/main.log b/outputs/2026-06-04/06-22-28/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-04/06-24-27/.hydra/config.yaml b/outputs/2026-06-04/06-24-27/.hydra/config.yaml deleted file mode 100644 index 8e50746541fe21844337576173118a055f92a686..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-27/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-24-27/.hydra/hydra.yaml b/outputs/2026-06-04/06-24-27/.hydra/hydra.yaml deleted file mode 100644 index 4a64186936bf540d7cda57a16db90faa1560ce8e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-27/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-24-27 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-24-27/.hydra/overrides.yaml b/outputs/2026-06-04/06-24-27/.hydra/overrides.yaml deleted file mode 100644 index 9827984995b3535ee762f6a1370ef381be13c82e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-27/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-24-27/main.log b/outputs/2026-06-04/06-24-27/main.log deleted file mode 100644 index 9d93adadbd86ace21ba8e1c5fabe75373d22ec88..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-27/main.log +++ /dev/null @@ -1,77 +0,0 @@ -[2026-06-04 06:24:58,261][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-04 06:24:58,374][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-04 06:24:58,617][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-04 06:24:58,844][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-04 06:25:01,130][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-04 06:25:01,209][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-04 06:25:01,432][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-04 06:25:01,930][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-04 06:25:02,129][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-04 06:25:02,209][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-04 06:25:03,309][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-04 06:25:07,333][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:20,837][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:21,907][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 199 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -911 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,573.841 Total estimated model params size (MB) -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-04 06:25:22,815][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,016][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-07 06:24:48,021][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-04/06-24-28/.hydra/config.yaml b/outputs/2026-06-04/06-24-28/.hydra/config.yaml deleted file mode 100644 index 8e50746541fe21844337576173118a055f92a686..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-28/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-24-28/.hydra/hydra.yaml b/outputs/2026-06-04/06-24-28/.hydra/hydra.yaml deleted file mode 100644 index 5db771ecded429b21a919779b187c22a9754c67d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-28/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-24-28 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-24-28/.hydra/overrides.yaml b/outputs/2026-06-04/06-24-28/.hydra/overrides.yaml deleted file mode 100644 index 9827984995b3535ee762f6a1370ef381be13c82e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-28/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-24-28/main.log b/outputs/2026-06-04/06-24-28/main.log deleted file mode 100644 index 0a28a589b244b086011356493a633db24faacc7b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-24-28/main.log +++ /dev/null @@ -1,7 +0,0 @@ -[2026-06-04 06:25:01,236][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-04 06:25:20,836][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-04 06:25:22,816][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-07 06:24:48,015][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-07 06:24:48,020][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-04/06-56-07/.hydra/config.yaml b/outputs/2026-06-04/06-56-07/.hydra/config.yaml deleted file mode 100644 index a4ac400fccef6e5a5d7e37db8dbefb5a01135211..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-07/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - memory_condition_length: 0 - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-56-07/.hydra/hydra.yaml b/outputs/2026-06-04/06-56-07/.hydra/hydra.yaml deleted file mode 100644 index 76aef61d23a699b88bde5806bbde2768b034f705..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-07/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-56-07 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-56-07/.hydra/overrides.yaml b/outputs/2026-06-04/06-56-07/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-07/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-56-07/main.log b/outputs/2026-06-04/06-56-07/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-04/06-56-08/.hydra/config.yaml b/outputs/2026-06-04/06-56-08/.hydra/config.yaml deleted file mode 100644 index a4ac400fccef6e5a5d7e37db8dbefb5a01135211..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-08/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - memory_condition_length: 0 - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-56-08/.hydra/hydra.yaml b/outputs/2026-06-04/06-56-08/.hydra/hydra.yaml deleted file mode 100644 index bdccbcee233f9548be589ba7ae1bd8dd90840782..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-08/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-56-08 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-56-08/.hydra/overrides.yaml b/outputs/2026-06-04/06-56-08/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-08/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-56-08/main.log b/outputs/2026-06-04/06-56-08/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-04/06-56-09/.hydra/config.yaml b/outputs/2026-06-04/06-56-09/.hydra/config.yaml deleted file mode 100644 index a4ac400fccef6e5a5d7e37db8dbefb5a01135211..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-09/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - memory_condition_length: 0 - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-04/06-56-09/.hydra/hydra.yaml b/outputs/2026-06-04/06-56-09/.hydra/hydra.yaml deleted file mode 100644 index 70af697fb90bc595008a410b8a8033183b5b8cb1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-09/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-04/06-56-09 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-04/06-56-09/.hydra/overrides.yaml b/outputs/2026-06-04/06-56-09/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-04/06-56-09/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-04/06-56-09/main.log b/outputs/2026-06-04/06-56-09/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-05/04-16-56/.hydra/config.yaml b/outputs/2026-06-05/04-16-56/.hydra/config.yaml deleted file mode 100644 index 3640bb3cbb3cb04f1b3f8269c7c813f70d37cc96..0000000000000000000000000000000000000000 --- a/outputs/2026-06-05/04-16-56/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-05/04-16-56/.hydra/hydra.yaml b/outputs/2026-06-05/04-16-56/.hydra/hydra.yaml deleted file mode 100644 index 768c93711a10688babfde34234b9a1b65faefc2c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-05/04-16-56/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-05/04-16-56 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-05/04-16-56/.hydra/overrides.yaml b/outputs/2026-06-05/04-16-56/.hydra/overrides.yaml deleted file mode 100644 index f5cfcc49c373d20f7b9d19b1aa4a76e9f3840859..0000000000000000000000000000000000000000 --- a/outputs/2026-06-05/04-16-56/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_separate_memory_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-05/04-16-56/main.log b/outputs/2026-06-05/04-16-56/main.log deleted file mode 100644 index 39aa75d8d27fd8e02bf5092d9941485f93727450..0000000000000000000000000000000000000000 --- a/outputs/2026-06-05/04-16-56/main.log +++ /dev/null @@ -1,68 +0,0 @@ -[2026-06-05 04:17:39,878][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-05 04:17:40,248][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-05 04:17:40,249][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-05 04:17:40,249][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-05 04:17:40,249][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-05 04:17:40,249][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-05 04:17:43,881][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-05 04:17:44,806][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-05 04:17:47,641][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-05 04:17:47,647][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-05 04:17:47,654][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-05 04:17:54,247][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,626][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,625][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:08,626][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-05 04:18:11,315][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 631 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -632 M Trainable params -231 M Non-trainable params -864 M Total params -3,456.995 Total estimated model params size (MB) -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-05 04:18:13,299][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-08 04:16:57,058][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-08 04:16:57,058][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-08 04:16:57,058][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-08 04:16:57,058][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,058][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-08 04:16:57,059][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-08 04:16:57,057][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-08 04:16:57,059][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-07/16-35-26/.hydra/config.yaml b/outputs/2026-06-07/16-35-26/.hydra/config.yaml deleted file mode 100644 index 8e50746541fe21844337576173118a055f92a686..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-26/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-07/16-35-26/.hydra/hydra.yaml b/outputs/2026-06-07/16-35-26/.hydra/hydra.yaml deleted file mode 100644 index 81a82156d72c3752fbe5c603233054b88bea8007..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-26/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-07/16-35-26 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-07/16-35-26/.hydra/overrides.yaml b/outputs/2026-06-07/16-35-26/.hydra/overrides.yaml deleted file mode 100644 index 9827984995b3535ee762f6a1370ef381be13c82e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-26/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-07/16-35-26/main.log b/outputs/2026-06-07/16-35-26/main.log deleted file mode 100644 index 30116287f976764ee9144b1593522dc290d1fb31..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-26/main.log +++ /dev/null @@ -1,41 +0,0 @@ -[2026-06-07 16:36:13,566][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-07 16:36:13,664][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-07 16:36:13,664][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-07 16:36:13,664][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-07 16:36:13,664][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-07 16:36:13,665][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-07 16:36:14,033][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-07 16:36:14,933][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-07 16:36:15,013][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-07 16:36:15,983][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-07 16:36:22,895][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-07 16:36:36,654][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/checkpoints/last.ckpt -[2026-06-07 16:36:54,144][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:54,798][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:54,867][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:54,942][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:55,760][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 199 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -911 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,573.841 Total estimated model params size (MB) -[2026-06-07 16:36:56,564][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/checkpoints/last.ckpt -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 05:47:16,693][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=350000` reached. diff --git a/outputs/2026-06-07/16-35-27/.hydra/config.yaml b/outputs/2026-06-07/16-35-27/.hydra/config.yaml deleted file mode 100644 index 8e50746541fe21844337576173118a055f92a686..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-27/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-07/16-35-27/.hydra/hydra.yaml b/outputs/2026-06-07/16-35-27/.hydra/hydra.yaml deleted file mode 100644 index a09cc5cf14121c8febd7d6fac46b73f7e298ab85..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-27/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-07/16-35-27 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-07/16-35-27/.hydra/overrides.yaml b/outputs/2026-06-07/16-35-27/.hydra/overrides.yaml deleted file mode 100644 index 9827984995b3535ee762f6a1370ef381be13c82e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-27/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-07/16-35-27/main.log b/outputs/2026-06-07/16-35-27/main.log deleted file mode 100644 index bfa392e49802ad5dbddbf3928a9965c464f1e981..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-27/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-07 16:36:13,993][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-07 16:36:14,015][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-07 16:36:54,140][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:54,886][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-07/16-35-28/.hydra/config.yaml b/outputs/2026-06-07/16-35-28/.hydra/config.yaml deleted file mode 100644 index 8e50746541fe21844337576173118a055f92a686..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-28/.hydra/config.yaml +++ /dev/null @@ -1,232 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-07/16-35-28/.hydra/hydra.yaml b/outputs/2026-06-07/16-35-28/.hydra/hydra.yaml deleted file mode 100644 index aa5fb6c56cc4651d1ebe01b7967edfcbff87e1c8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-28/.hydra/hydra.yaml +++ /dev/null @@ -1,230 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-07/16-35-28 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-07/16-35-28/.hydra/overrides.yaml b/outputs/2026-06-07/16-35-28/.hydra/overrides.yaml deleted file mode 100644 index 9827984995b3535ee762f6a1370ef381be13c82e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-28/.hydra/overrides.yaml +++ /dev/null @@ -1,72 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-07/16-35-28/main.log b/outputs/2026-06-07/16-35-28/main.log deleted file mode 100644 index b17c008e8543df07bf454a62108560684f4e8f94..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/16-35-28/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-07 16:36:14,069][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-07 16:36:14,085][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-07 16:36:54,051][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:54,868][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 16:36:59,258][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-07/21-26-49/.hydra/config.yaml b/outputs/2026-06-07/21-26-49/.hydra/config.yaml deleted file mode 100644 index 383e38a44d3cc4e0ab79c0c959f3bc58ccd515ee..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-49/.hydra/config.yaml +++ /dev/null @@ -1,227 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 1024 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-07/21-26-49/.hydra/hydra.yaml b/outputs/2026-06-07/21-26-49/.hydra/hydra.yaml deleted file mode 100644 index a13ec33a69ac59d582598dc9fe88637e82fa108e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-49/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=null - - ++algorithm.dememwm.injection.memory_hidden_size=1024 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=1024,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=null,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-07/21-26-49 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-07/21-26-49/.hydra/overrides.yaml b/outputs/2026-06-07/21-26-49/.hydra/overrides.yaml deleted file mode 100644 index fd80489eda9c1b11b112f3919f467eb50f635c86..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-49/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=null -- ++algorithm.dememwm.injection.memory_hidden_size=1024 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-07/21-26-49/main.log b/outputs/2026-06-07/21-26-49/main.log deleted file mode 100644 index 817132fc528bebf40fed79b60c28215472161e42..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-49/main.log +++ /dev/null @@ -1,43 +0,0 @@ -[2026-06-07 21:27:32,831][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-07 21:27:33,847][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-07 21:27:33,847][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-07 21:27:33,847][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-07 21:27:33,847][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-07 21:27:33,848][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-07 21:27:36,265][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-07 21:27:37,179][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-07 21:27:39,857][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-07 21:27:39,859][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-07 21:27:39,866][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-07 21:27:46,350][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:01,607][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 994 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 1.8 M -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -997 M Trainable params -231 M Non-trainable params -1.2 B Total params -4,915.943 Total estimated model params size (MB) -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 21:26:49,109][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-10 21:26:49,109][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-07/21-26-50/.hydra/config.yaml b/outputs/2026-06-07/21-26-50/.hydra/config.yaml deleted file mode 100644 index 383e38a44d3cc4e0ab79c0c959f3bc58ccd515ee..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-50/.hydra/config.yaml +++ /dev/null @@ -1,227 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 1024 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-07/21-26-50/.hydra/hydra.yaml b/outputs/2026-06-07/21-26-50/.hydra/hydra.yaml deleted file mode 100644 index 10d01f33aad1dc3f65878fa482cbb5660b84b8ac..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-50/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=null - - ++algorithm.dememwm.injection.memory_hidden_size=1024 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=1024,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=null,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-07/21-26-50 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-07/21-26-50/.hydra/overrides.yaml b/outputs/2026-06-07/21-26-50/.hydra/overrides.yaml deleted file mode 100644 index fd80489eda9c1b11b112f3919f467eb50f635c86..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-50/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_separate_memory_full_hidden1024_all_layers_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=null -- ++algorithm.dememwm.injection.memory_hidden_size=1024 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-07/21-26-50/main.log b/outputs/2026-06-07/21-26-50/main.log deleted file mode 100644 index 2dc8e7ef9c5225bce71131954893bf79d870bfb4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-07/21-26-50/main.log +++ /dev/null @@ -1,25 +0,0 @@ -[2026-06-07 21:27:39,859][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-07 21:27:39,859][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-07 21:27:39,859][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-07 21:27:39,863][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-07 21:27:39,866][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:00,514][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-07 21:28:05,410][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-10 21:26:49,110][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-10 21:26:49,111][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-10/18-59-16/.hydra/config.yaml b/outputs/2026-06-10/18-59-16/.hydra/config.yaml deleted file mode 100644 index 147c36b0f4d0828be9a12297959c5d927b0c80fa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-16/.hydra/config.yaml +++ /dev/null @@ -1,233 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - learn_gate: false - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-10/18-59-16/.hydra/hydra.yaml b/outputs/2026-06-10/18-59-16/.hydra/hydra.yaml deleted file mode 100644 index e27f2e3cb518e7f3445395eb78ec2cc5331d6f28..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-16/.hydra/hydra.yaml +++ /dev/null @@ -1,231 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.learn_gate=false - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.learn_gate=false,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-10/18-59-16 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-10/18-59-16/.hydra/overrides.yaml b/outputs/2026-06-10/18-59-16/.hydra/overrides.yaml deleted file mode 100644 index 65ff4ced4ac2e99c0a14f6cfa4b22f1224db9db3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-16/.hydra/overrides.yaml +++ /dev/null @@ -1,73 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.learn_gate=false -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-10/18-59-16/main.log b/outputs/2026-06-10/18-59-16/main.log deleted file mode 100644 index 6bb750a03107f6b3592ebb33d8985cc5aaf7c96c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-16/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-10 19:00:07,918][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-10 19:00:07,959][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-10 19:00:07,964][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-10/18-59-17/.hydra/config.yaml b/outputs/2026-06-10/18-59-17/.hydra/config.yaml deleted file mode 100644 index 147c36b0f4d0828be9a12297959c5d927b0c80fa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-17/.hydra/config.yaml +++ /dev/null @@ -1,233 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - learn_gate: false - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-10/18-59-17/.hydra/hydra.yaml b/outputs/2026-06-10/18-59-17/.hydra/hydra.yaml deleted file mode 100644 index 7b18c3d3b8f0ea108de3d68602cdeb2a49debad1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-17/.hydra/hydra.yaml +++ /dev/null @@ -1,231 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.learn_gate=false - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.learn_gate=false,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-10/18-59-17 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-10/18-59-17/.hydra/overrides.yaml b/outputs/2026-06-10/18-59-17/.hydra/overrides.yaml deleted file mode 100644 index 65ff4ced4ac2e99c0a14f6cfa4b22f1224db9db3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-17/.hydra/overrides.yaml +++ /dev/null @@ -1,73 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.learn_gate=false -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-10/18-59-17/main.log b/outputs/2026-06-10/18-59-17/main.log deleted file mode 100644 index 30968524713c99b578fe767957a32a7b637bc90c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-17/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-10 19:00:07,944][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-10 19:00:07,946][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-10 19:00:07,954][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 18:59:31,412][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-13 18:59:31,415][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-10/18-59-18/.hydra/config.yaml b/outputs/2026-06-10/18-59-18/.hydra/config.yaml deleted file mode 100644 index 147c36b0f4d0828be9a12297959c5d927b0c80fa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-18/.hydra/config.yaml +++ /dev/null @@ -1,233 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - memory_condition_length: 0 - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - memory_condition_length: 0 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - token_patch_size: 2 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 3 - dynamic: - enabled: true - exclude_latest_local_frames: 4 - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - revisit: - enabled: true - learn_gate: false - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - high_quality_fov_threshold: 0.7 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 3 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - dit_hidden_size: 1024 - anchor_gate: 1.0 - dynamic_gate: 1.0 - revisit_gate: 1.0 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-10/18-59-18/.hydra/hydra.yaml b/outputs/2026-06-10/18-59-18/.hydra/hydra.yaml deleted file mode 100644 index 94cd061e40d7bc335ae3890f9146bf829f336b6e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-18/.hydra/hydra.yaml +++ /dev/null @@ -1,231 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.memory_condition_length=0 - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=3 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.learn_gate=false - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=3 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=3,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.exclude_latest_local_frames=4,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=3,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.learn_gate=false,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.memory_condition_length=0,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-10/18-59-18 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-10/18-59-18/.hydra/overrides.yaml b/outputs/2026-06-10/18-59-18/.hydra/overrides.yaml deleted file mode 100644 index 65ff4ced4ac2e99c0a14f6cfa4b22f1224db9db3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-18/.hydra/overrides.yaml +++ /dev/null @@ -1,73 +0,0 @@ -- +name=train_dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_full_berzelius_8a100_bs16_global128_350k_gate_off/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.memory_condition_length=0 -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=3 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.exclude_latest_local_frames=4 -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.learn_gate=false -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=3 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-10/18-59-18/main.log b/outputs/2026-06-10/18-59-18/main.log deleted file mode 100644 index ffcb3ae7933baf3d09563a85455e78aa86dd1052..0000000000000000000000000000000000000000 --- a/outputs/2026-06-10/18-59-18/main.log +++ /dev/null @@ -1,38 +0,0 @@ -[2026-06-10 19:00:00,071][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-10 19:00:00,241][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-10 19:00:00,241][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-10 19:00:00,241][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-10 19:00:00,241][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-10 19:00:00,242][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-10 19:00:04,346][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-10 19:00:05,350][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-10 19:00:07,918][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-10 19:00:07,935][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-10 19:00:14,848][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:29,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-10 19:00:30,556][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 199 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -911 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,573.841 Total estimated model params size (MB) -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-10 19:00:36,026][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 18:59:31,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-13 18:59:31,414][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-13 18:59:31,415][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-13 18:59:31,415][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-47-56/.hydra/config.yaml b/outputs/2026-06-12/00-47-56/.hydra/config.yaml deleted file mode 100644 index 64fffe9c620215d1cab65a6f5f95ac635e29032a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-56/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-47-56/.hydra/hydra.yaml b/outputs/2026-06-12/00-47-56/.hydra/hydra.yaml deleted file mode 100644 index a96e636112690ae0aab2b0cbb4fdee97873e5955..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-56/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-47-56 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-47-56/.hydra/overrides.yaml b/outputs/2026-06-12/00-47-56/.hydra/overrides.yaml deleted file mode 100644 index f4f2bf0676c18a5d49b9d2724af9d00ea079c01f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-56/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-47-56/main.log b/outputs/2026-06-12/00-47-56/main.log deleted file mode 100644 index 25dd9d818a2d38c2ef03f781679ab947afaac5c0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-56/main.log +++ /dev/null @@ -1,92 +0,0 @@ -[2026-06-12 00:48:44,382][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 00:48:44,549][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 00:48:44,549][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 00:48:44,549][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 00:48:44,549][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 00:48:44,549][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 00:48:47,035][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:48:48,020][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:48:50,539][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 00:48:50,540][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 00:48:50,541][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 00:48:50,569][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 00:48:50,576][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 00:48:57,704][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:15,948][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | dememwm_memory_pose_embed | Sequential | 266 K -6 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,273.565 Total estimated model params size (MB) -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 03:47:19,003][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_pairs', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,013][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_revisit_frames', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,013][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,013][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,014][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,014][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,014][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_minus_non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,015][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,015][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,015][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,015][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,016][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,016][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,016][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,017][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,017][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,017][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,017][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:47:19,018][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-15 00:47:49,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 00:47:49,314][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 00:47:49,314][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 00:47:49,315][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-47-57/.hydra/config.yaml b/outputs/2026-06-12/00-47-57/.hydra/config.yaml deleted file mode 100644 index 64fffe9c620215d1cab65a6f5f95ac635e29032a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-57/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-47-57/.hydra/hydra.yaml b/outputs/2026-06-12/00-47-57/.hydra/hydra.yaml deleted file mode 100644 index a6d19d055ae57f3903a601e2b5a69ad99971c706..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-57/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-47-57 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-47-57/.hydra/overrides.yaml b/outputs/2026-06-12/00-47-57/.hydra/overrides.yaml deleted file mode 100644 index f4f2bf0676c18a5d49b9d2724af9d00ea079c01f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-57/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_plucker_mem_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-47-57/main.log b/outputs/2026-06-12/00-47-57/main.log deleted file mode 100644 index 3f8966bab1404718bbc2fc08332e93fcfc7dd3f4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-47-57/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-12 00:48:50,540][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 00:48:50,540][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 00:48:50,540][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:14,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:18,953][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 00:47:49,311][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 00:47:49,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:47:49,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 00:47:49,315][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-48-24/.hydra/config.yaml b/outputs/2026-06-12/00-48-24/.hydra/config.yaml deleted file mode 100644 index 0d9cad3f5777e026cecb2ef35be0972cd5179996..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-24/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: dit_patch - downsample_ratio: 1 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-48-24/.hydra/hydra.yaml b/outputs/2026-06-12/00-48-24/.hydra/hydra.yaml deleted file mode 100644 index 4e157f91d8cb55e4abf17577f47dec6de52d9b35..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-24/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=dit_patch - - ++algorithm.dememwm.revisit.compress.downsample_ratio=1 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=1,++algorithm.dememwm.revisit.compress.tokenization=dit_patch,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-48-24 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-48-24/.hydra/overrides.yaml b/outputs/2026-06-12/00-48-24/.hydra/overrides.yaml deleted file mode 100644 index 3fa71fb829a9374240e37cf84ade058f611e3b71..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-24/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=dit_patch -- ++algorithm.dememwm.revisit.compress.downsample_ratio=1 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-48-24/main.log b/outputs/2026-06-12/00-48-24/main.log deleted file mode 100644 index f1404cc929fdb52f266332807e2f3538ee118826..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-24/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-12 00:49:22,766][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:49,278][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 00:48:19,413][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-48-25/.hydra/config.yaml b/outputs/2026-06-12/00-48-25/.hydra/config.yaml deleted file mode 100644 index aa51822707170f2c75fc5292f065c9648969e23c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-25/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-48-25/.hydra/hydra.yaml b/outputs/2026-06-12/00-48-25/.hydra/hydra.yaml deleted file mode 100644 index 97133726d4390a9b2a93d3087c34421de30a1b52..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-25/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-48-25 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-48-25/.hydra/overrides.yaml b/outputs/2026-06-12/00-48-25/.hydra/overrides.yaml deleted file mode 100644 index c4014a7d99ff6a70b2b8f002c479c5b7f21978a1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-25/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-48-25/main.log b/outputs/2026-06-12/00-48-25/main.log deleted file mode 100644 index 7c191da84eefd6a7239f20b60ea5c921ac767d6a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-25/main.log +++ /dev/null @@ -1,106 +0,0 @@ -[2026-06-12 00:49:16,635][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 00:49:16,821][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 00:49:16,821][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 00:49:16,821][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 00:49:16,821][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 00:49:16,821][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 00:49:19,282][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:49:20,227][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:49:22,749][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 00:49:22,762][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 00:49:22,763][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 00:49:22,765][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 00:49:22,770][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 00:49:22,778][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 00:49:22,801][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 00:49:28,124][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,278][lightning.pytorch.trainer.connectors.signal_c 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,272.513 Total estimated model params size (MB) -[2026-06-12 00:49:49,278][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:49,279][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 03:44:30,912][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_pairs', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,922][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_revisit_frames', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,922][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,922][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,922][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,922][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,923][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_minus_non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,923][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,923][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,923][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,924][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,924][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,924][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,924][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,924][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:44:30,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 00:48:19,028][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 00:48:19,115][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,119][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,406][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:20,380][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:20,386][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:20,391][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 00:48:19,030][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,030][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 00:48:19,121][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-48-26/.hydra/config.yaml b/outputs/2026-06-12/00-48-26/.hydra/config.yaml deleted file mode 100644 index aa51822707170f2c75fc5292f065c9648969e23c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-26/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-48-26/.hydra/hydra.yaml b/outputs/2026-06-12/00-48-26/.hydra/hydra.yaml deleted file mode 100644 index 39a61532c360651afa124af25f4f6a5f614901ef..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-26/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-48-26 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-48-26/.hydra/overrides.yaml b/outputs/2026-06-12/00-48-26/.hydra/overrides.yaml deleted file mode 100644 index c4014a7d99ff6a70b2b8f002c479c5b7f21978a1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-26/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_lr8e5_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-48-26/main.log b/outputs/2026-06-12/00-48-26/main.log deleted file mode 100644 index 79a76c8a5bac7bf8d7b6264891fcc5bf389b0522..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-48-26/main.log +++ /dev/null @@ -1,79 +0,0 @@ -[2026-06-12 00:49:15,159][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 00:49:15,353][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 00:49:15,353][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 00:49:15,353][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 00:49:15,353][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 00:49:15,353][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 00:49:18,355][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:49:19,243][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:49:21,767][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 00:49:21,775][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 00:49:27,831][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 00:49:43,209][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:43,209][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:49:44,184][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | dememwm_memory_pose_embed | Sequential | 266 K -6 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,273.565 Total estimated model params size (MB) -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:49:47,277][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 03:48:30,553][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_pairs', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,361][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_revisit_frames', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,362][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,362][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,362][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_minus_non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,364][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,364][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,364][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,364][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,365][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,365][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,365][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,365][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 03:48:31,366][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-15 00:48:19,030][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,029][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 00:48:19,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 00:48:19,036][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:48:19,799][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-49-54/.hydra/config.yaml b/outputs/2026-06-12/00-49-54/.hydra/config.yaml deleted file mode 100644 index 55d20637a0061a8b4a79eab4fbd188633caa8bbb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-54/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-49-54/.hydra/hydra.yaml b/outputs/2026-06-12/00-49-54/.hydra/hydra.yaml deleted file mode 100644 index 34eba634fa847e5c8e40447ac5fbd772d1f34eda..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-54/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-49-54 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-49-54/.hydra/overrides.yaml b/outputs/2026-06-12/00-49-54/.hydra/overrides.yaml deleted file mode 100644 index 80caf7ad361473e24bf1e74f22bf1d873b276fbe..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-54/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-49-54/main.log b/outputs/2026-06-12/00-49-54/main.log deleted file mode 100644 index 16aee1c7d8486b71cb2020f5bbdc293deec5e70d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-54/main.log +++ /dev/null @@ -1,92 +0,0 @@ -[2026-06-12 00:50:39,167][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 00:50:39,538][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 00:50:39,538][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 00:50:39,538][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 00:50:39,538][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 00:50:39,539][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 00:50:43,264][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:50:44,227][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 00:50:52,936][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:08,475][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | dememwm_memory_pose_embed | Sequential | 266 K -6 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,273.565 Total estimated model params size (MB) -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,478][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 07:47:02,486][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_pairs', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,381][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_revisit_frames', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,381][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,382][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,382][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,382][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,383][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_minus_non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,383][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,383][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,383][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-12 07:47:03,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-15 00:49:48,753][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 00:49:48,754][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,754][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 00:49:48,754][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,756][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/00-49-55/.hydra/config.yaml b/outputs/2026-06-12/00-49-55/.hydra/config.yaml deleted file mode 100644 index 55d20637a0061a8b4a79eab4fbd188633caa8bbb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-55/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/00-49-55/.hydra/hydra.yaml b/outputs/2026-06-12/00-49-55/.hydra/hydra.yaml deleted file mode 100644 index ae1347d6c54c8a55dedbc4898f1a7951a3d4916d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-55/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/00-49-55 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/00-49-55/.hydra/overrides.yaml b/outputs/2026-06-12/00-49-55/.hydra/overrides.yaml deleted file mode 100644 index 80caf7ad361473e24bf1e74f22bf1d873b276fbe..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-55/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-12/00-49-55/main.log b/outputs/2026-06-12/00-49-55/main.log deleted file mode 100644 index d1382db40f796e0c9b60ef0641fed449454c40f4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/00-49-55/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 00:50:46,806][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 00:50:46,807][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:06,875][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,477][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 00:51:09,478][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 00:49:48,753][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 00:49:48,755][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 00:49:48,753][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 00:49:48,757][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/09-13-36/.hydra/config.yaml b/outputs/2026-06-12/09-13-36/.hydra/config.yaml deleted file mode 100644 index b47e3332559c53ed999c275a717b07d1982311f8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-36/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/09-13-36/.hydra/hydra.yaml b/outputs/2026-06-12/09-13-36/.hydra/hydra.yaml deleted file mode 100644 index b45a7c94f520e55c7be18b19ae5865a470b99e34..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-36/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=[3,7,11,15] - - ++algorithm.dememwm.injection.memory_hidden_size=512 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=512,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=[3,7,11,15],algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/09-13-36 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/09-13-36/.hydra/overrides.yaml b/outputs/2026-06-12/09-13-36/.hydra/overrides.yaml deleted file mode 100644 index 8ee3288e706e04827ca476ead0592fea9c288ffa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-36/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=[3,7,11,15] -- ++algorithm.dememwm.injection.memory_hidden_size=512 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/09-13-36/main.log b/outputs/2026-06-12/09-13-36/main.log deleted file mode 100644 index 4a33adeba034db40570e99dc4501678f76cc3f5f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-36/main.log +++ /dev/null @@ -1,77 +0,0 @@ -[2026-06-12 09:14:21,877][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 09:14:21,992][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 09:14:21,993][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 09:14:21,993][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 09:14:21,993][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 09:14:21,993][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 09:14:24,614][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 09:14:25,556][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 09:14:28,274][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 09:14:34,383][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 09:14:48,621][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:50,305][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 681 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -682 M Trainable params -231 M Non-trainable params -914 M Total params -3,658.543 Total estimated model params size (MB) -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 09:13:48,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,036][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 09:13:48,036][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 09:13:48,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 09:13:48,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,042][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 09:13:48,042][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,043][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 09:13:48,043][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/09-13-37/.hydra/config.yaml b/outputs/2026-06-12/09-13-37/.hydra/config.yaml deleted file mode 100644 index b47e3332559c53ed999c275a717b07d1982311f8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-37/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/09-13-37/.hydra/hydra.yaml b/outputs/2026-06-12/09-13-37/.hydra/hydra.yaml deleted file mode 100644 index 5832f21c92f34fd5a59de0c06a8710d85595e17a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-37/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=[3,7,11,15] - - ++algorithm.dememwm.injection.memory_hidden_size=512 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=512,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=[3,7,11,15],algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/09-13-37 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/09-13-37/.hydra/overrides.yaml b/outputs/2026-06-12/09-13-37/.hydra/overrides.yaml deleted file mode 100644 index 8ee3288e706e04827ca476ead0592fea9c288ffa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-37/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=[3,7,11,15] -- ++algorithm.dememwm.injection.memory_hidden_size=512 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/09-13-37/main.log b/outputs/2026-06-12/09-13-37/main.log deleted file mode 100644 index ef2f4e3653893acee7d9a33fcc4dec4894caaf7f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/09-13-37/main.log +++ /dev/null @@ -1,7 +0,0 @@ -[2026-06-12 09:14:28,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 09:14:48,622][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 09:14:51,296][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 09:13:48,033][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 09:13:48,034][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 09:13:48,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 09:13:48,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/19-27-06/.hydra/config.yaml b/outputs/2026-06-12/19-27-06/.hydra/config.yaml deleted file mode 100644 index 0407b139e7d545e5e741be674e569dea342b0ccc..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-06/.hydra/config.yaml +++ /dev/null @@ -1,227 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 1024 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/19-27-06/.hydra/hydra.yaml b/outputs/2026-06-12/19-27-06/.hydra/hydra.yaml deleted file mode 100644 index 4bc2ba09df8e9cd3062832277abcf52f3b78570f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-06/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=null - - ++algorithm.dememwm.injection.memory_hidden_size=1024 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=1024,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=null,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/19-27-06 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/19-27-06/.hydra/overrides.yaml b/outputs/2026-06-12/19-27-06/.hydra/overrides.yaml deleted file mode 100644 index 4cb6f8a5aeb033b8dea3cbf193d9bd78ce5d0b0b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-06/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=null -- ++algorithm.dememwm.injection.memory_hidden_size=1024 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/19-27-06/main.log b/outputs/2026-06-12/19-27-06/main.log deleted file mode 100644 index 102dd2f9b4cd3e131da305bb037d9b691f869baa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-06/main.log +++ /dev/null @@ -1,53 +0,0 @@ -[2026-06-12 19:27:53,749][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 19:27:54,699][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 19:27:54,699][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 19:27:54,699][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 19:27:54,699][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 19:27:54,700][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 19:27:57,364][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 19:27:58,336][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 19:28:01,170][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 19:28:01,171][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 19:28:01,184][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 19:28:01,186][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 19:28:01,214][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 19:28:06,789][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:23,087][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 1.8 M -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 591 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 591 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.2 B Total params -4,983.118 Total estimated model params size (MB) -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 19:28:24,096][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 19:27:23,331][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/19-27-07/.hydra/config.yaml b/outputs/2026-06-12/19-27-07/.hydra/config.yaml deleted file mode 100644 index 0407b139e7d545e5e741be674e569dea342b0ccc..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-07/.hydra/config.yaml +++ /dev/null @@ -1,227 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 1024 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/19-27-07/.hydra/hydra.yaml b/outputs/2026-06-12/19-27-07/.hydra/hydra.yaml deleted file mode 100644 index 4d5a789479e8a19a80c6168dcc93961975417d43..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-07/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=null - - ++algorithm.dememwm.injection.memory_hidden_size=1024 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=1024,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=null,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/19-27-07 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/19-27-07/.hydra/overrides.yaml b/outputs/2026-06-12/19-27-07/.hydra/overrides.yaml deleted file mode 100644 index 4cb6f8a5aeb033b8dea3cbf193d9bd78ce5d0b0b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-07/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=null -- ++algorithm.dememwm.injection.memory_hidden_size=1024 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/19-27-07/main.log b/outputs/2026-06-12/19-27-07/main.log deleted file mode 100644 index f22b75e24b37d29186b119ddee629c41d93efcda..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-07/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-12 19:28:01,175][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 19:28:01,185][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/19-27-08/.hydra/config.yaml b/outputs/2026-06-12/19-27-08/.hydra/config.yaml deleted file mode 100644 index 0407b139e7d545e5e741be674e569dea342b0ccc..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-08/.hydra/config.yaml +++ /dev/null @@ -1,227 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 1024 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/19-27-08/.hydra/hydra.yaml b/outputs/2026-06-12/19-27-08/.hydra/hydra.yaml deleted file mode 100644 index 8926ceccd522a17c9d099df6b63e4ca5958005c7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-08/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=null - - ++algorithm.dememwm.injection.memory_hidden_size=1024 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=1024,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=null,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/19-27-08 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/19-27-08/.hydra/overrides.yaml b/outputs/2026-06-12/19-27-08/.hydra/overrides.yaml deleted file mode 100644 index 4cb6f8a5aeb033b8dea3cbf193d9bd78ce5d0b0b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-08/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_alllayers1024_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=null -- ++algorithm.dememwm.injection.memory_hidden_size=1024 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/19-27-08/main.log b/outputs/2026-06-12/19-27-08/main.log deleted file mode 100644 index 77e2ea4ebf7cf63e135bb8299f31481c202af4fd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/19-27-08/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-12 19:28:01,187][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 19:28:21,673][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 19:28:24,095][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 19:27:23,332][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 19:27:23,333][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-29-00/.hydra/config.yaml b/outputs/2026-06-12/20-29-00/.hydra/config.yaml deleted file mode 100644 index c604bc2394aaab5ec435addbaf49b8ad5d6093c2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-00/.hydra/config.yaml +++ /dev/null @@ -1,230 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: learned - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-29-00/.hydra/hydra.yaml b/outputs/2026-06-12/20-29-00/.hydra/hydra.yaml deleted file mode 100644 index 900382e1f4ee5605101ff311f67612e03f840701..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-00/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=learned - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=learned,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-29-00 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-29-00/.hydra/overrides.yaml b/outputs/2026-06-12/20-29-00/.hydra/overrides.yaml deleted file mode 100644 index c5f3775fa2a65fb67c1ac07a2800951036ac581a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-00/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=learned -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-29-00/main.log b/outputs/2026-06-12/20-29-00/main.log deleted file mode 100644 index 031f1442ccf02665b5705a8c830564b41f6f6430..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-00/main.log +++ /dev/null @@ -1,25 +0,0 @@ -[2026-06-12 20:29:51,095][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 20:29:51,096][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 20:29:51,096][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 20:29:51,097][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 20:29:51,102][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:28:55,038][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 20:28:55,038][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 20:28:55,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 20:28:55,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 20:28:55,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 20:28:55,269][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:28:55,275][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:28:55,282][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:28:56,289][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:28:57,242][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-29-02/.hydra/config.yaml b/outputs/2026-06-12/20-29-02/.hydra/config.yaml deleted file mode 100644 index c604bc2394aaab5ec435addbaf49b8ad5d6093c2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-02/.hydra/config.yaml +++ /dev/null @@ -1,230 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: learned - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-29-02/.hydra/hydra.yaml b/outputs/2026-06-12/20-29-02/.hydra/hydra.yaml deleted file mode 100644 index 6a54788a2de76a737bb0802b04a31c5008f40c6e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-02/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=learned - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=learned,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-29-02 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-29-02/.hydra/overrides.yaml b/outputs/2026-06-12/20-29-02/.hydra/overrides.yaml deleted file mode 100644 index c5f3775fa2a65fb67c1ac07a2800951036ac581a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-02/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=learned -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-29-02/main.log b/outputs/2026-06-12/20-29-02/main.log deleted file mode 100644 index f73b1074d1dd3156b62ff3236188165402710675..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-02/main.log +++ /dev/null @@ -1,33 +0,0 @@ -[2026-06-12 20:29:43,958][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 20:29:44,123][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 20:29:44,123][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 20:29:44,123][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 20:29:44,124][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 20:29:44,124][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 20:29:47,170][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:29:48,143][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:29:51,096][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 20:29:57,625][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:14,990][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,271.812 Total estimated model params size (MB) -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:28:55,038][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 20:28:55,265][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-29-03/.hydra/config.yaml b/outputs/2026-06-12/20-29-03/.hydra/config.yaml deleted file mode 100644 index c604bc2394aaab5ec435addbaf49b8ad5d6093c2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-03/.hydra/config.yaml +++ /dev/null @@ -1,230 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: learned - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-29-03/.hydra/hydra.yaml b/outputs/2026-06-12/20-29-03/.hydra/hydra.yaml deleted file mode 100644 index 4704299eef82eabbbc25f9b3280540a8119ae833..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-03/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=learned - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=learned,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-29-03 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-29-03/.hydra/overrides.yaml b/outputs/2026-06-12/20-29-03/.hydra/overrides.yaml deleted file mode 100644 index c5f3775fa2a65fb67c1ac07a2800951036ac581a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-03/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_base_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_base_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=learned -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-29-03/main.log b/outputs/2026-06-12/20-29-03/main.log deleted file mode 100644 index 16122ddee07fc02f4393bbb4a59e4efee869a25b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-29-03/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-12 20:29:51,097][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 20:29:51,097][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:12,541][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:30:16,982][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:28:55,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 20:28:55,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 20:28:55,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:28:56,330][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-31-35/.hydra/config.yaml b/outputs/2026-06-12/20-31-35/.hydra/config.yaml deleted file mode 100644 index 31a98d1e56829de50924f50c9e233702aa6798d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-35/.hydra/config.yaml +++ /dev/null @@ -1,237 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: dit_patch - downsample_ratio: null - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: dit_patch - downsample_ratio: null - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: false - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-31-35/.hydra/hydra.yaml b/outputs/2026-06-12/20-31-35/.hydra/hydra.yaml deleted file mode 100644 index 2051f3e54083ee28c5b99434be25319420e7da72..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-35/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=dit_patch - - ++algorithm.dememwm.anchor.compress.downsample_ratio=null - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=dit_patch - - ++algorithm.dememwm.revisit.compress.downsample_ratio=null - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1],++algorithm.dememwm.anchor.compress.downsample_ratio=null,++algorithm.dememwm.anchor.compress.tokenization=dit_patch,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=null,++algorithm.dememwm.revisit.compress.tokenization=dit_patch,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-31-35 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-31-35/.hydra/overrides.yaml b/outputs/2026-06-12/20-31-35/.hydra/overrides.yaml deleted file mode 100644 index 0ce4a4f18540b037c78822c163519003195c0134..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-35/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_nocomp_a2_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=dit_patch -- ++algorithm.dememwm.anchor.compress.downsample_ratio=null -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=dit_patch -- ++algorithm.dememwm.revisit.compress.downsample_ratio=null -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-31-35/main.log b/outputs/2026-06-12/20-31-35/main.log deleted file mode 100644 index 13b4d2650e12a049bd0d7246f63d8d85b69781d8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-35/main.log +++ /dev/null @@ -1,70 +0,0 @@ -[2026-06-12 20:32:21,967][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 20:32:22,094][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 20:32:22,094][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 20:32:22,095][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 20:32:22,095][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 20:32:22,095][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 20:32:25,319][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:32:26,291][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:32:28,875][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 20:32:28,876][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 20:32:28,876][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 20:32:28,876][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 20:32:28,876][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 20:32:28,876][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 20:32:34,395][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:47,524][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:49,416][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | DiTPatchMemoryProjector | 33.3 K -3 | dememwm_revisit_proj | DiTPatchMemoryProjector | 33.3 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -835 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,269.707 Total estimated model params size (MB) -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:53,400][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:25,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 20:31:25,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 20:31:25,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 20:31:25,041][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,040][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 20:31:25,043][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,045][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 20:31:25,045][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,045][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 20:31:25,045][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 20:31:25,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-31-36/.hydra/config.yaml b/outputs/2026-06-12/20-31-36/.hydra/config.yaml deleted file mode 100644 index 37657208acae3cddc363fbf170f94749a1917b00..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-36/.hydra/config.yaml +++ /dev/null @@ -1,239 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-31-36/.hydra/hydra.yaml b/outputs/2026-06-12/20-31-36/.hydra/hydra.yaml deleted file mode 100644 index c3b2bab34f7d924562b8438f41396e6b40e81455..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-36/.hydra/hydra.yaml +++ /dev/null @@ -1,241 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-31-36 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-31-36/.hydra/overrides.yaml b/outputs/2026-06-12/20-31-36/.hydra/overrides.yaml deleted file mode 100644 index 67a4320747586094d17d9591a2df050cfadcf719..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-36/.hydra/overrides.yaml +++ /dev/null @@ -1,83 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-31-36/main.log b/outputs/2026-06-12/20-31-36/main.log deleted file mode 100644 index 08be6205e59a7e2b64cf2e5d6bb0600981804376..0000000000000000000000000000000000000000 Binary files a/outputs/2026-06-12/20-31-36/main.log and /dev/null differ diff --git a/outputs/2026-06-12/20-31-39/.hydra/config.yaml b/outputs/2026-06-12/20-31-39/.hydra/config.yaml deleted file mode 100644 index 969b5c001b3c5895df04636acb7922a0506e92c3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-39/.hydra/config.yaml +++ /dev/null @@ -1,239 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: false - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-31-39/.hydra/hydra.yaml b/outputs/2026-06-12/20-31-39/.hydra/hydra.yaml deleted file mode 100644 index 9b4d2d34cdb8273078dbddc36de3b145113ffb8f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-39/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-31-39 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-31-39/.hydra/overrides.yaml b/outputs/2026-06-12/20-31-39/.hydra/overrides.yaml deleted file mode 100644 index 3ea991f8fd7f385f8fc1231d04a8448120510272..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-39/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-31-39/main.log b/outputs/2026-06-12/20-31-39/main.log deleted file mode 100644 index 9686260614187eacbb62aefe4a9a2a349fe316e1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-39/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-12 20:32:29,964][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:25,200][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-31-45/.hydra/config.yaml b/outputs/2026-06-12/20-31-45/.hydra/config.yaml deleted file mode 100644 index 969b5c001b3c5895df04636acb7922a0506e92c3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-45/.hydra/config.yaml +++ /dev/null @@ -1,239 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: false - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-31-45/.hydra/hydra.yaml b/outputs/2026-06-12/20-31-45/.hydra/hydra.yaml deleted file mode 100644 index c44d1a1bdf01dc9230cf986389ba02353bea0dcb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-45/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-31-45 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-31-45/.hydra/overrides.yaml b/outputs/2026-06-12/20-31-45/.hydra/overrides.yaml deleted file mode 100644 index 3ea991f8fd7f385f8fc1231d04a8448120510272..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-45/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-31-45/main.log b/outputs/2026-06-12/20-31-45/main.log deleted file mode 100644 index 13858e904261245f9a058eb4ff77013098599163..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-45/main.log +++ /dev/null @@ -1,25 +0,0 @@ -[2026-06-12 20:32:29,945][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-12 20:32:29,945][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-12 20:32:29,945][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-12 20:32:29,964][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-12 20:32:29,970][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:25,200][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,204][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-15 20:31:25,205][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-12/20-31-46/.hydra/config.yaml b/outputs/2026-06-12/20-31-46/.hydra/config.yaml deleted file mode 100644 index 969b5c001b3c5895df04636acb7922a0506e92c3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-46/.hydra/config.yaml +++ /dev/null @@ -1,239 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 1100 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: false - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-12/20-31-46/.hydra/hydra.yaml b/outputs/2026-06-12/20-31-46/.hydra/hydra.yaml deleted file mode 100644 index e92c1a63e16e25d7088d893888852d5a10976428..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-46/.hydra/hydra.yaml +++ /dev/null @@ -1,233 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=1100 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=1100,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-12/20-31-46 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-12/20-31-46/.hydra/overrides.yaml b/outputs/2026-06-12/20-31-46/.hydra/overrides.yaml deleted file mode 100644 index 3ea991f8fd7f385f8fc1231d04a8448120510272..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-46/.hydra/overrides.yaml +++ /dev/null @@ -1,75 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=1100 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-12/20-31-46/main.log b/outputs/2026-06-12/20-31-46/main.log deleted file mode 100644 index 90743b27d7b605f7523254275ae25e3c0765fb14..0000000000000000000000000000000000000000 --- a/outputs/2026-06-12/20-31-46/main.log +++ /dev/null @@ -1,38 +0,0 @@ -[2026-06-12 20:32:22,623][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-12 20:32:22,956][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-12 20:32:22,956][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-12 20:32:22,956][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-12 20:32:22,956][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-12 20:32:22,957][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-12 20:32:26,261][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:32:27,232][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-12 20:32:29,945][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-12 20:32:29,955][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-12 20:32:36,204][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:52,650][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-12 20:32:53,913][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,271.812 Total estimated model params size (MB) -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-12 20:32:55,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-15 20:31:25,201][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-15 20:31:25,202][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-13/01-38-54/.hydra/config.yaml b/outputs/2026-06-13/01-38-54/.hydra/config.yaml deleted file mode 100644 index 7600f66abc165be3f3a30297a5a7491e9cf9caa4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-54/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 1 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-13/01-38-54/.hydra/hydra.yaml b/outputs/2026-06-13/01-38-54/.hydra/hydra.yaml deleted file mode 100644 index 404847c914dd4bf6edfb8e8e4285f00e0dd33b42..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-54/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=1 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=1,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-13/01-38-54 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-13/01-38-54/.hydra/overrides.yaml b/outputs/2026-06-13/01-38-54/.hydra/overrides.yaml deleted file mode 100644 index 038f66a14d8d4ea292ed8a55c1a0997b8c913eb3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-54/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=1 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-13/01-38-54/main.log b/outputs/2026-06-13/01-38-54/main.log deleted file mode 100644 index 33e4416fff6513375a2892ddb14af1cc7b7208a7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-54/main.log +++ /dev/null @@ -1,116 +0,0 @@ -[2026-06-13 01:39:56,295][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-13 01:39:56,494][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-13 01:39:56,494][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-13 01:39:56,494][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-13 01:39:56,494][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-13 01:39:56,495][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-13 01:40:00,499][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-13 01:40:01,436][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-13 01:40:03,862][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-13 01:40:03,879][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-13 01:40:03,889][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-13 01:40:03,890][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-13 01:40:03,896][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-13 01:40:03,896][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-13 01:40:03,904][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-13 01:40:10,484][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:26,159][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | dememwm_memory_pose_embed | Sequential | 266 K -6 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,273.565 Total estimated model params size (MB) -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-13 04:41:16,354][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_pairs', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,372][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_revisit_frames', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_minus_non_revisit_lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,374][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,374][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,374][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,374][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_8_32', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_32_128', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/num_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,376][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,376][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/self_consistency_psnr_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-13 04:41:16,376][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('revisit/revisit_lpips_gap_128_inf', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,045][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-16 01:38:57,048][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,048][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,050][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-16 01:38:57,050][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-16 01:38:57,050][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,050][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,050][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-13/01-38-55/.hydra/config.yaml b/outputs/2026-06-13/01-38-55/.hydra/config.yaml deleted file mode 100644 index 7600f66abc165be3f3a30297a5a7491e9cf9caa4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-55/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 1 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: true - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-13/01-38-55/.hydra/hydra.yaml b/outputs/2026-06-13/01-38-55/.hydra/hydra.yaml deleted file mode 100644 index 6a77ea7057dd383b54ebe380a66755aa71ebda3f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-55/.hydra/hydra.yaml +++ /dev/null @@ -1,249 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=1 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=1,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-13/01-38-55 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-13/01-38-55/.hydra/overrides.yaml b/outputs/2026-06-13/01-38-55/.hydra/overrides.yaml deleted file mode 100644 index 038f66a14d8d4ea292ed8a55c1a0997b8c913eb3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-55/.hydra/overrides.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_rev1cap_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=1 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-13/01-38-55/main.log b/outputs/2026-06-13/01-38-55/main.log deleted file mode 100644 index 0802811f68dfb661f2005dfb734bfd71e5fa2d37..0000000000000000000000000000000000000000 --- a/outputs/2026-06-13/01-38-55/main.log +++ /dev/null @@ -1,7 +0,0 @@ -[2026-06-13 01:40:03,888][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-13 01:40:25,178][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-13 01:40:29,954][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:38:57,046][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-16 01:38:57,047][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-16 01:38:57,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-45/.hydra/config.yaml b/outputs/2026-06-15/18-54-45/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-45/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-45/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-45/.hydra/hydra.yaml deleted file mode 100644 index d6489daca5f2280a41c2d9facbe0a40a1e97532a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-45/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-45 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-45/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-45/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-45/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-45/main.log b/outputs/2026-06-15/18-54-45/main.log deleted file mode 100644 index 78652dc24e484ffccbc117d71efba82744e7f978..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-45/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 18:55:40,325][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-18 18:54:55,539][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-46/.hydra/config.yaml b/outputs/2026-06-15/18-54-46/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-46/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-46/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-46/.hydra/hydra.yaml deleted file mode 100644 index 500f46ad7faeeb43beb397debf3dfd94821f641e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-46/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-46 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-46/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-46/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-46/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-46/main.log b/outputs/2026-06-15/18-54-46/main.log deleted file mode 100644 index 0c59c6caf974a12750e4dec4f4e77c66307f3cfb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-46/main.log +++ /dev/null @@ -1,29 +0,0 @@ -[2026-06-15 18:55:34,415][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-15 18:55:34,576][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-15 18:55:34,576][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-15 18:55:34,576][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-15 18:55:34,576][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-15 18:55:34,577][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-15 18:55:36,899][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 18:55:37,814][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 18:55:40,345][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-15 18:55:46,981][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:02,540][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-18 18:54:55,543][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-47/.hydra/config.yaml b/outputs/2026-06-15/18-54-47/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-47/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-47/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-47/.hydra/hydra.yaml deleted file mode 100644 index 22f622543b8aa950cd9e19d9508044c0b0ec26cc..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-47/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-47 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-47/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-47/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-47/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-47/main.log b/outputs/2026-06-15/18-54-47/main.log deleted file mode 100644 index 086e36b2b6b682eeff96136b83608c073c8d494a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-47/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-15 18:55:40,325][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-15 18:55:40,326][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-18 18:54:55,084][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-18 18:54:55,536][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 18:54:55,549][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-48/.hydra/config.yaml b/outputs/2026-06-15/18-54-48/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-48/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-48/.hydra/hydra.yaml deleted file mode 100644 index cb9eee5c85841d9b8421fdcc0cffbaafd76d7003..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-48/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-48/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-48/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-48/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-48/main.log b/outputs/2026-06-15/18-54-48/main.log deleted file mode 100644 index 4e614f6a9f22b96266f008be2eb96b598f340b2b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-48/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 18:55:40,322][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-18 18:54:55,170][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-49/.hydra/config.yaml b/outputs/2026-06-15/18-54-49/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-49/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-49/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-49/.hydra/hydra.yaml deleted file mode 100644 index ce4a6e4a135bfa1f2a80f20c9de785e1f463e6b7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-49/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-49 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-49/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-49/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-49/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-49/main.log b/outputs/2026-06-15/18-54-49/main.log deleted file mode 100644 index 10a2d31176363ecdba195ae2f2fd035a74a2dae0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-49/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 18:55:40,319][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-18 18:54:56,547][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-50/.hydra/config.yaml b/outputs/2026-06-15/18-54-50/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-50/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-50/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-50/.hydra/hydra.yaml deleted file mode 100644 index 04987ac61bb0f795dab3866ca2199b4b00f61bd3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-50/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-50 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-50/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-50/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-50/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-50/main.log b/outputs/2026-06-15/18-54-50/main.log deleted file mode 100644 index f9970fc043398e811154d2915fccd073deae6438..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-50/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 18:55:40,310][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-18 18:54:56,536][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-54-51/.hydra/config.yaml b/outputs/2026-06-15/18-54-51/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-51/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-54-51/.hydra/hydra.yaml b/outputs/2026-06-15/18-54-51/.hydra/hydra.yaml deleted file mode 100644 index 32d84637392175f9151371cd42d30f455c806948..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-51/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-54-51 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-54-51/.hydra/overrides.yaml b/outputs/2026-06-15/18-54-51/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-51/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-54-51/main.log b/outputs/2026-06-15/18-54-51/main.log deleted file mode 100644 index 921b1dc9c65de510c6bb3458a1c33f5ac1b10296..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-54-51/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 18:55:40,302][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-15 18:56:01,422][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 18:56:05,241][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:54:55,083][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-18 18:54:56,520][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-48/.hydra/config.yaml b/outputs/2026-06-15/18-59-48/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-48/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-48/.hydra/hydra.yaml deleted file mode 100644 index 3d619fc106f61669f27b1401d755485797319f13..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-48/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-48/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-48/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-48/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-48/main.log b/outputs/2026-06-15/18-59-48/main.log deleted file mode 100644 index 27dfa969ccc0efd5d196fa1dab2e499d8ec1be83..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-48/main.log +++ /dev/null @@ -1,12 +0,0 @@ -[2026-06-15 19:00:42,132][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-15 19:00:42,154][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,084][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-18 18:59:55,085][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-49/.hydra/config.yaml b/outputs/2026-06-15/18-59-49/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-49/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-49/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-49/.hydra/hydra.yaml deleted file mode 100644 index ccf9be1a20e78a9cb8b761f46521e8425b0392e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-49/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-49 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-49/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-49/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-49/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-49/main.log b/outputs/2026-06-15/18-59-49/main.log deleted file mode 100644 index e5b97f42409c66d44bc7dc6fd50e4b512cbbabb6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-49/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-15 19:00:42,135][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-15 19:00:42,140][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,762][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 18:59:55,762][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-51/.hydra/config.yaml b/outputs/2026-06-15/18-59-51/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-51/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-51/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-51/.hydra/hydra.yaml deleted file mode 100644 index 5970397648a4d951ed6c427c2b470dc6e90549cd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-51/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-51 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-51/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-51/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-51/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-51/main.log b/outputs/2026-06-15/18-59-51/main.log deleted file mode 100644 index e7117f86b163d4e67f49ac6828c46ebcebb46ddc..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-51/main.log +++ /dev/null @@ -1,29 +0,0 @@ -[2026-06-15 19:00:35,281][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-15 19:00:36,030][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-15 19:00:36,030][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-15 19:00:36,030][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-15 19:00:36,030][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-15 19:00:36,031][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-15 19:00:38,673][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 19:00:39,574][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 19:00:42,126][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-15 19:00:47,753][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:05,638][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-18 18:59:55,764][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-52/.hydra/config.yaml b/outputs/2026-06-15/18-59-52/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-52/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-52/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-52/.hydra/hydra.yaml deleted file mode 100644 index 5a627322315cda0a95c201feb284d216a706d3c0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-52/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-52 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-52/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-52/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-52/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-52/main.log b/outputs/2026-06-15/18-59-52/main.log deleted file mode 100644 index dd9452ac66f3289f2bc1349800c09724b8de6476..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-52/main.log +++ /dev/null @@ -1,7 +0,0 @@ -[2026-06-15 19:00:42,126][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,084][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-18 18:59:55,085][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 18:59:55,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-18 18:59:55,091][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-53/.hydra/config.yaml b/outputs/2026-06-15/18-59-53/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-53/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-53/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-53/.hydra/hydra.yaml deleted file mode 100644 index 38a6f84f1c5cea13107f15e3b75cc87c6da85cf3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-53/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-53 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-53/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-53/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-53/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-53/main.log b/outputs/2026-06-15/18-59-53/main.log deleted file mode 100644 index 051742d540ecbe47cab5f2b5b384161b64aae01a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-53/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 19:00:42,148][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-18 18:59:55,765][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/18-59-54/.hydra/config.yaml b/outputs/2026-06-15/18-59-54/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-54/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/18-59-54/.hydra/hydra.yaml b/outputs/2026-06-15/18-59-54/.hydra/hydra.yaml deleted file mode 100644 index f70ca6fff3e606dc33c70d461d57d0c6ee88042d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-54/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/18-59-54 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/18-59-54/.hydra/overrides.yaml b/outputs/2026-06-15/18-59-54/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-54/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/18-59-54/main.log b/outputs/2026-06-15/18-59-54/main.log deleted file mode 100644 index f516c41b588f466ce6636287a8c8270ea1d4681e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/18-59-54/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 19:00:42,153][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-15 19:01:03,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 19:01:07,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 18:59:55,763][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-18 18:59:55,764][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-30-31/.hydra/config.yaml b/outputs/2026-06-15/20-30-31/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-31/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-30-31/.hydra/hydra.yaml b/outputs/2026-06-15/20-30-31/.hydra/hydra.yaml deleted file mode 100644 index 89fe5db3e4d8ff3f27c00e14326f063f2799fbec..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-31/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-30-31 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-30-31/.hydra/overrides.yaml b/outputs/2026-06-15/20-30-31/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-31/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-30-31/main.log b/outputs/2026-06-15/20-30-31/main.log deleted file mode 100644 index 455f835303d18d24b4a715eb15abb1170e92a6d4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-31/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 20:31:13,328][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-30-32/.hydra/config.yaml b/outputs/2026-06-15/20-30-32/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-32/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-30-32/.hydra/hydra.yaml b/outputs/2026-06-15/20-30-32/.hydra/hydra.yaml deleted file mode 100644 index 9214e682f5c217507508ee41e56f422d51683216..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-32/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-30-32 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-30-32/.hydra/overrides.yaml b/outputs/2026-06-15/20-30-32/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-32/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-30-32/main.log b/outputs/2026-06-15/20-30-32/main.log deleted file mode 100644 index d11e282333abeeafc9f173c235ab4eb74edec3ce..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-32/main.log +++ /dev/null @@ -1,20 +0,0 @@ -[2026-06-15 20:31:12,814][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-15 20:31:12,985][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-15 20:31:13,040][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-15 20:31:13,161][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-15 20:31:32,963][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:30:27,142][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-30-33/.hydra/config.yaml b/outputs/2026-06-15/20-30-33/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-33/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-30-33/.hydra/hydra.yaml b/outputs/2026-06-15/20-30-33/.hydra/hydra.yaml deleted file mode 100644 index ff71af1e7f20cd9f620261eff4567d321cad48c0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-33/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-30-33 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-30-33/.hydra/overrides.yaml b/outputs/2026-06-15/20-30-33/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-33/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-30-33/main.log b/outputs/2026-06-15/20-30-33/main.log deleted file mode 100644 index ada057e89673c58693817ae477256192f3843256..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-33/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 20:31:12,918][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-30-34/.hydra/config.yaml b/outputs/2026-06-15/20-30-34/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-34/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-30-34/.hydra/hydra.yaml b/outputs/2026-06-15/20-30-34/.hydra/hydra.yaml deleted file mode 100644 index e578a581d5a0783b911e9dff63bebe078331ec97..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-34/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-30-34 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-30-34/.hydra/overrides.yaml b/outputs/2026-06-15/20-30-34/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-34/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-30-34/main.log b/outputs/2026-06-15/20-30-34/main.log deleted file mode 100644 index 2ee9adeaafe1fa3276f89c0b899a8ad1de1faee7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-30-34/main.log +++ /dev/null @@ -1,34 +0,0 @@ -[2026-06-15 20:31:09,908][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-15 20:31:10,112][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-15 20:31:10,112][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-15 20:31:10,112][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-15 20:31:10,112][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-15 20:31:10,113][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-15 20:31:10,335][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 20:31:10,525][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 20:31:12,711][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-15 20:31:13,990][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-15 20:31:19,195][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-15 20:31:32,964][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:32,963][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:31:34,433][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:31:36,431][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-18 20:30:27,143][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:30:27,144][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-34/.hydra/config.yaml b/outputs/2026-06-15/20-32-34/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-34/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-34/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-34/.hydra/hydra.yaml deleted file mode 100644 index d8528b05f1ae37b987d0de492b20b8ab58a3f10c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-34/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-34 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-34/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-34/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-34/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-34/main.log b/outputs/2026-06-15/20-32-34/main.log deleted file mode 100644 index ca441f4e1723eddaeffddea2231a0990c4861405..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-34/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-15 20:33:23,114][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-15 20:33:23,115][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,490][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-18 20:32:27,491][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:32:27,491][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-35/.hydra/config.yaml b/outputs/2026-06-15/20-32-35/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-35/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-35/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-35/.hydra/hydra.yaml deleted file mode 100644 index 5e0ed0bb12185ffcdfbaf0333ef97814ed26762e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-35/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-35 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-35/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-35/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-35/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-35/main.log b/outputs/2026-06-15/20-32-35/main.log deleted file mode 100644 index 846fb91582c5bd72d40a0a20b9638b39f72d03cf..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-35/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 20:33:23,739][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,491][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-36/.hydra/config.yaml b/outputs/2026-06-15/20-32-36/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-36/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-36/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-36/.hydra/hydra.yaml deleted file mode 100644 index 8a332677b086c6f65020062bacef5bf8bb296a8d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-36/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-36 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-36/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-36/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-36/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-36/main.log b/outputs/2026-06-15/20-32-36/main.log deleted file mode 100644 index c329cb223ef3e3548083f717429678ef74dbbd57..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-36/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-15 20:33:23,114][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-15 20:33:23,133][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,490][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-18 20:32:27,490][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-37/.hydra/config.yaml b/outputs/2026-06-15/20-32-37/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-37/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-37/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-37/.hydra/hydra.yaml deleted file mode 100644 index 26faedda0f5f9af0b52fb4105b5c65481e53f301..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-37/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-37 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-37/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-37/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-37/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-37/main.log b/outputs/2026-06-15/20-32-37/main.log deleted file mode 100644 index 603952866cb5a1c01ef47452f7135316fd4016b5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-37/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 20:33:23,114][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-18 20:32:27,493][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-38/.hydra/config.yaml b/outputs/2026-06-15/20-32-38/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-38/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-38/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-38/.hydra/hydra.yaml deleted file mode 100644 index d9d11eb0fd771b92d05a7b188eeb4b8b0dec1081..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-38/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-38 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-38/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-38/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-38/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-38/main.log b/outputs/2026-06-15/20-32-38/main.log deleted file mode 100644 index b78e91d5d7b9b220f44e049ce497f04270fe7f0f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-38/main.log +++ /dev/null @@ -1,29 +0,0 @@ -[2026-06-15 20:33:16,765][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-15 20:33:16,898][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-15 20:33:16,898][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-15 20:33:16,898][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-15 20:33:16,898][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-15 20:33:16,899][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-15 20:33:19,199][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 20:33:20,394][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-15 20:33:23,139][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-15 20:33:29,340][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:43,567][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-18 20:32:27,493][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-15/20-32-39/.hydra/config.yaml b/outputs/2026-06-15/20-32-39/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-39/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-15/20-32-39/.hydra/hydra.yaml b/outputs/2026-06-15/20-32-39/.hydra/hydra.yaml deleted file mode 100644 index 198cc560030c04bc59cd5521e30748ce75993f01..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-39/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-15/20-32-39 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-15/20-32-39/.hydra/overrides.yaml b/outputs/2026-06-15/20-32-39/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-39/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-15/20-32-39/main.log b/outputs/2026-06-15/20-32-39/main.log deleted file mode 100644 index e2e3746ad1c7856d42130c22d61fea2b1d257308..0000000000000000000000000000000000000000 --- a/outputs/2026-06-15/20-32-39/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-15 20:33:23,115][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-15 20:33:42,350][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-15 20:33:47,591][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-18 20:32:27,491][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-18 20:32:27,492][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-19/.hydra/config.yaml b/outputs/2026-06-16/00-48-19/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-19/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-19/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-19/.hydra/hydra.yaml deleted file mode 100644 index aa5c29d08e1b53e2bbffe68441bf2dcfe31ba9b7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-19/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-19 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-19/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-19/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-19/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-19/main.log b/outputs/2026-06-16/00-48-19/main.log deleted file mode 100644 index b74c63b1be0667da941566c665bb39f9ca92ad4f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-19/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-16 00:49:13,101][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-19 00:48:30,247][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-21/.hydra/config.yaml b/outputs/2026-06-16/00-48-21/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-21/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-21/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-21/.hydra/hydra.yaml deleted file mode 100644 index 8cfe2c304c28c836df53aac2beb0b83b34636238..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-21/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-21 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-21/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-21/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-21/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-21/main.log b/outputs/2026-06-16/00-48-21/main.log deleted file mode 100644 index af31a22113a1c1ed595ea2a8b7ddd242112deb03..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-21/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-16 00:49:13,100][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,079][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-19 00:48:30,234][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-22/.hydra/config.yaml b/outputs/2026-06-16/00-48-22/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-22/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-22/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-22/.hydra/hydra.yaml deleted file mode 100644 index 7bfe670eafac9ab2023648587771f36c401ae420..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-22/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-22 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-22/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-22/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-22/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-22/main.log b/outputs/2026-06-16/00-48-22/main.log deleted file mode 100644 index ba34ddaff4120898601f6183c68c2099cf3e8c44..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-22/main.log +++ /dev/null @@ -1,34 +0,0 @@ -[2026-06-16 00:49:06,036][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-16 00:49:06,170][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-16 00:49:06,171][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-16 00:49:06,171][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-16 00:49:06,171][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-16 00:49:06,172][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-16 00:49:09,011][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-16 00:49:10,152][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-16 00:49:13,099][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-16 00:49:13,109][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-16 00:49:18,823][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:35,840][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,079][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-19 00:48:29,168][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 00:48:29,259][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-23/.hydra/config.yaml b/outputs/2026-06-16/00-48-23/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-23/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-23/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-23/.hydra/hydra.yaml deleted file mode 100644 index 9201ccd74144c5924f8aff31bd6879699bac5a7b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-23/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-23 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-23/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-23/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-23/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-23/main.log b/outputs/2026-06-16/00-48-23/main.log deleted file mode 100644 index 188f3becabb2d2f72d3b3b6ab29e670bedbee0a1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-23/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-16 00:49:13,100][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-16 00:49:13,127][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-19 00:48:30,236][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 00:48:30,255][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-24/.hydra/config.yaml b/outputs/2026-06-16/00-48-24/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-24/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-24/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-24/.hydra/hydra.yaml deleted file mode 100644 index 4de5e6405e5b0cbdf39fe514789e015370235a63..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-24/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-24 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-24/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-24/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-24/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-24/main.log b/outputs/2026-06-16/00-48-24/main.log deleted file mode 100644 index 02999ed3bd877431fb8da996ce0b0fe874c9a2a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-24/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-16 00:49:13,098][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-19 00:48:30,249][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/00-48-25/.hydra/config.yaml b/outputs/2026-06-16/00-48-25/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-25/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/00-48-25/.hydra/hydra.yaml b/outputs/2026-06-16/00-48-25/.hydra/hydra.yaml deleted file mode 100644 index fd5290bdfc8f05d6b5907aa817d27f8b73c3cce5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-25/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/00-48-25 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/00-48-25/.hydra/overrides.yaml b/outputs/2026-06-16/00-48-25/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-25/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-16/00-48-25/main.log b/outputs/2026-06-16/00-48-25/main.log deleted file mode 100644 index 44e4ec1b3beb723704a2f58dc95f3e136a2d5131..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/00-48-25/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-16 00:49:13,125][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-16 00:49:33,682][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 00:49:39,817][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 00:48:29,080][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-19 00:48:29,245][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/01-22-05/.hydra/config.yaml b/outputs/2026-06-16/01-22-05/.hydra/config.yaml deleted file mode 100644 index d448e9f3d5b0f919cdd9db5960181d659ab5f07c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-05/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/01-22-05/.hydra/hydra.yaml b/outputs/2026-06-16/01-22-05/.hydra/hydra.yaml deleted file mode 100644 index fdb58455d25e31211f6884ce8fc19a4295bd6c77..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-05/.hydra/hydra.yaml +++ /dev/null @@ -1,250 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/01-22-05 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/01-22-05/.hydra/overrides.yaml b/outputs/2026-06-16/01-22-05/.hydra/overrides.yaml deleted file mode 100644 index de99b40af9c811edc833b8fa7d40eac9948a8baa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-05/.hydra/overrides.yaml +++ /dev/null @@ -1,92 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-16/01-22-05/main.log b/outputs/2026-06-16/01-22-05/main.log deleted file mode 100644 index 621c56d94e3f06f89a6238bf82b69921141b63b4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-05/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-16 01:22:51,951][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-16 01:22:52,671][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-16 01:22:54,087][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-16 01:23:35,952][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:35,985][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:36,330][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 01:21:59,077][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-19 01:21:59,078][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-19 01:21:59,078][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-19 01:22:00,081][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:22:01,013][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:22:01,993][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-16/01-22-06/.hydra/config.yaml b/outputs/2026-06-16/01-22-06/.hydra/config.yaml deleted file mode 100644 index d448e9f3d5b0f919cdd9db5960181d659ab5f07c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-06/.hydra/config.yaml +++ /dev/null @@ -1,262 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 4 - max_epochs: -1 - max_steps: 750000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 10000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: null - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 10000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - training_sampling: - revisit_biased_prob: 0.3 - min_gap: 32 - candidate_starts: 8 - train_negatives: - enabled: true - prob: 0.1 - start_step: 20000 - branches: - - wrong_pose - - time_shuffle - - source_matched_random - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - gate_mode: none - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - tokenization: spatial_conv_pool - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 20000 - ramp_steps: 20000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - snr_shifted_memory: - enabled: true - mode: step_offset - offset_steps: 1 - apply_anchor: true - apply_dynamic: false - apply_revisit: true - noise_per_target: true - mark_as_generated: false - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - injection: - memory_hidden_size: 512 - pose_aware: true - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-16/01-22-06/.hydra/hydra.yaml b/outputs/2026-06-16/01-22-06/.hydra/hydra.yaml deleted file mode 100644 index 6da994d16f159663689d367c40724a6d68dc498c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-06/.hydra/hydra.yaml +++ /dev/null @@ -1,250 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.injection.pose_aware=true - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=20000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.snr_shifted_memory.enabled=true - - ++algorithm.dememwm.snr_shifted_memory.mode=step_offset - - ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 - - ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true - - ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false - - ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true - - ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true - - ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false - - ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 - - ++algorithm.dememwm.training_sampling.min_gap=32 - - ++algorithm.dememwm.train_negatives.enabled=true - - ++algorithm.dememwm.train_negatives.prob=0.10 - - ++algorithm.dememwm.train_negatives.start_step=20000 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.gate_mode=none - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=10000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=4 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=10000 - - experiment.training.max_steps=750000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=10000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=20000,++algorithm.dememwm.generated_history_proxy.start_step=20000,++algorithm.dememwm.injection.pose_aware=true,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.gate_mode=none,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.dememwm.snr_shifted_memory.apply_anchor=true,++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false,++algorithm.dememwm.snr_shifted_memory.apply_revisit=true,++algorithm.dememwm.snr_shifted_memory.enabled=true,++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false,++algorithm.dememwm.snr_shifted_memory.mode=step_offset,++algorithm.dememwm.snr_shifted_memory.noise_per_target=true,++algorithm.dememwm.snr_shifted_memory.offset_steps=1,++algorithm.dememwm.train_negatives.enabled=true,++algorithm.dememwm.train_negatives.prob=0.10,++algorithm.dememwm.train_negatives.start_step=20000,++algorithm.dememwm.training_sampling.min_gap=32,++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=4,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=750000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=10000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-16/01-22-06 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-16/01-22-06/.hydra/overrides.yaml b/outputs/2026-06-16/01-22-06/.hydra/overrides.yaml deleted file mode 100644 index de99b40af9c811edc833b8fa7d40eac9948a8baa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-06/.hydra/overrides.yaml +++ /dev/null @@ -1,92 +0,0 @@ -- +name=train_dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.injection.pose_aware=true -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=20000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=20000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.snr_shifted_memory.enabled=true -- ++algorithm.dememwm.snr_shifted_memory.mode=step_offset -- ++algorithm.dememwm.snr_shifted_memory.offset_steps=1 -- ++algorithm.dememwm.snr_shifted_memory.apply_anchor=true -- ++algorithm.dememwm.snr_shifted_memory.apply_dynamic=false -- ++algorithm.dememwm.snr_shifted_memory.apply_revisit=true -- ++algorithm.dememwm.snr_shifted_memory.noise_per_target=true -- ++algorithm.dememwm.snr_shifted_memory.mark_as_generated=false -- ++algorithm.dememwm.training_sampling.revisit_biased_prob=0.3 -- ++algorithm.dememwm.training_sampling.min_gap=32 -- ++algorithm.dememwm.train_negatives.enabled=true -- ++algorithm.dememwm.train_negatives.prob=0.10 -- ++algorithm.dememwm.train_negatives.start_step=20000 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.gate_mode=none -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.tokenization=spatial_conv_pool -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=10000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=4 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=10000 -- experiment.training.max_steps=750000 diff --git a/outputs/2026-06-16/01-22-06/main.log b/outputs/2026-06-16/01-22-06/main.log deleted file mode 100644 index 31646a7abc628c0749db0033787e08840872ebd9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-16/01-22-06/main.log +++ /dev/null @@ -1,54 +0,0 @@ -[2026-06-16 01:22:48,879][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-16 01:22:49,061][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-16 01:22:49,061][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-16 01:22:49,061][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-16 01:22:49,061][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-16 01:22:49,061][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-16 01:22:51,816][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-16 01:22:51,823][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-16 01:22:51,942][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-16 01:22:51,947][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-16 01:22:52,595][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-16 01:22:58,817][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-16 01:23:13,076][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/checkpoints/last.ckpt -[2026-06-16 01:23:35,880][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:36,308][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:36,308][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:36,329][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:36,352][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-16 01:23:39,909][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 834 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | dememwm_memory_pose_embed | Sequential | 266 K -6 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -7 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -836 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,273.565 Total estimated model params size (MB) -[2026-06-16 01:23:40,718][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/dememwm_sep_full_seq_gate_none_snr_shift_ntok8_8a100_bs4_global32_750k/checkpoints/last.ckpt -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-16 01:23:41,941][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 01:21:59,077][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-19 01:21:59,077][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-19 01:21:59,078][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-19 01:21:59,078][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-19 01:21:59,078][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-19 01:21:59,154][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:21:59,155][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:22:00,988][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:22:01,006][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 01:22:02,010][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/06-56-28/.hydra/config.yaml b/outputs/2026-06-19/06-56-28/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-28/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/06-56-28/.hydra/hydra.yaml b/outputs/2026-06-19/06-56-28/.hydra/hydra.yaml deleted file mode 100644 index c6a3c838718da06c801285736889ea28d017a2ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-28/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/06-56-28 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/06-56-28/.hydra/overrides.yaml b/outputs/2026-06-19/06-56-28/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-28/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/06-56-28/main.log b/outputs/2026-06-19/06-56-28/main.log deleted file mode 100644 index e6bc5345ee58f21825c11809f7f4d9733a6108c4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-28/main.log +++ /dev/null @@ -1,39 +0,0 @@ -[2026-06-19 06:57:13,669][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 06:57:13,810][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 06:57:13,810][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 06:57:13,810][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 06:57:13,810][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 06:57:13,811][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 06:57:16,523][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 06:57:16,524][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 06:57:17,008][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 06:57:23,488][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 06:57:38,413][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 06:58:01,475][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:02,077][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:02,100][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:03,288][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 06:58:04,450][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 06:58:06,696][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 06:56:31,692][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-22 06:56:31,692][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/06-56-29/.hydra/config.yaml b/outputs/2026-06-19/06-56-29/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-29/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/06-56-29/.hydra/hydra.yaml b/outputs/2026-06-19/06-56-29/.hydra/hydra.yaml deleted file mode 100644 index d48b394c02e701f93159597334b974e52b9d3012..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-29/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/06-56-29 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/06-56-29/.hydra/overrides.yaml b/outputs/2026-06-19/06-56-29/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-29/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/06-56-29/main.log b/outputs/2026-06-19/06-56-29/main.log deleted file mode 100644 index 7c30e9db4b34b2e227a84a44171068e555f3461e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-29/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-19 06:57:16,525][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 06:57:16,569][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 06:58:01,446][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:02,170][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-22 06:56:31,694][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-22 06:56:31,696][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/06-56-30/.hydra/config.yaml b/outputs/2026-06-19/06-56-30/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-30/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/06-56-30/.hydra/hydra.yaml b/outputs/2026-06-19/06-56-30/.hydra/hydra.yaml deleted file mode 100644 index 1ed6bfc555a6678cf3bbd6d204eb9b2ee181323f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-30/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/06-56-30 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/06-56-30/.hydra/overrides.yaml b/outputs/2026-06-19/06-56-30/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-30/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/06-56-30/main.log b/outputs/2026-06-19/06-56-30/main.log deleted file mode 100644 index 6330f1c22293ea31debf9e836ae6fd6b47e3be6a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-30/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 06:57:16,556][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 06:58:01,470][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:06,696][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 06:56:31,692][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/06-56-31/.hydra/config.yaml b/outputs/2026-06-19/06-56-31/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-31/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/06-56-31/.hydra/hydra.yaml b/outputs/2026-06-19/06-56-31/.hydra/hydra.yaml deleted file mode 100644 index b8c10ea3a879eb527c6aa94052e01d67de4d4fd1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-31/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/06-56-31 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/06-56-31/.hydra/overrides.yaml b/outputs/2026-06-19/06-56-31/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-31/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/06-56-31/main.log b/outputs/2026-06-19/06-56-31/main.log deleted file mode 100644 index 4ea7a7c87651a826aebb7a9cb0ec0dccb591a535..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-31/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 06:57:16,563][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 06:58:02,103][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 06:56:31,692][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/06-56-32/.hydra/config.yaml b/outputs/2026-06-19/06-56-32/.hydra/config.yaml deleted file mode 100644 index 8cf23fa199620eb0ba7a0c4a2c845205b684f882..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-32/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/06-56-32/.hydra/hydra.yaml b/outputs/2026-06-19/06-56-32/.hydra/hydra.yaml deleted file mode 100644 index 9170577acac6ee1ff1c8fea984915c226f173063..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-32/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/06-56-32 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/06-56-32/.hydra/overrides.yaml b/outputs/2026-06-19/06-56-32/.hydra/overrides.yaml deleted file mode 100644 index 4572a3376e9ca2fd0da2e5237fe57a6334cc6596..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-32/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/06-56-32/main.log b/outputs/2026-06-19/06-56-32/main.log deleted file mode 100644 index 52ecd14b522d7c50c809a87903a44d796957785c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/06-56-32/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 06:57:16,521][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 06:58:02,149][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 06:58:06,695][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 06:56:31,693][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-22 06:56:31,694][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-30/.hydra/config.yaml b/outputs/2026-06-19/12-46-30/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-30/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-30/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-30/.hydra/hydra.yaml deleted file mode 100644 index bb3bfc3be7ccb12eaa19678247b6d55de940be4f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-30/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-30 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-30/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-30/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-30/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-30/main.log b/outputs/2026-06-19/12-46-30/main.log deleted file mode 100644 index d2e134a4e16fd2d510e81708eca25eb4cf62a5cd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-30/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-19 12:47:21,735][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 12:47:21,814][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 12:48:13,581][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:13,598][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-22 12:46:34,526][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 12:46:35,544][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-31/.hydra/config.yaml b/outputs/2026-06-19/12-46-31/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-31/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-31/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-31/.hydra/hydra.yaml deleted file mode 100644 index 8e47ff562981b481cb496c37149af887201c547e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-31/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-31 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-31/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-31/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-31/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-31/main.log b/outputs/2026-06-19/12-46-31/main.log deleted file mode 100644 index 3b2e60b442124d19ba5052ede2555d1c480b5d43..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-31/main.log +++ /dev/null @@ -1,39 +0,0 @@ -[2026-06-19 12:47:18,197][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 12:47:21,157][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 12:47:21,284][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 12:47:21,284][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 12:47:21,284][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 12:47:21,285][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 12:47:21,285][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 12:47:22,686][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 12:47:23,441][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 12:47:29,445][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 12:47:45,097][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 12:48:12,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:12,797][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:13,619][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:15,122][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 12:48:16,709][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-22 12:46:33,052][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-22 12:46:33,145][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 12:46:33,506][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 12:46:33,540][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-32/.hydra/config.yaml b/outputs/2026-06-19/12-46-32/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-32/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-32/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-32/.hydra/hydra.yaml deleted file mode 100644 index 7bb0498eff38ac18041ffbe05e58ee626b1e3406..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-32/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-32 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-32/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-32/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-32/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-32/main.log b/outputs/2026-06-19/12-46-32/main.log deleted file mode 100644 index c15367432493af7819abc6f63cfa6a74b62f5228..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-32/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 12:47:21,437][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 12:48:12,771][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-22 12:46:34,527][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-33/.hydra/config.yaml b/outputs/2026-06-19/12-46-33/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-33/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-33/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-33/.hydra/hydra.yaml deleted file mode 100644 index fc48166484743a940ead651c521e67494962dcad..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-33/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-33 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-33/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-33/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-33/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-33/main.log b/outputs/2026-06-19/12-46-33/main.log deleted file mode 100644 index d9e2f76eb6ea4c75dc25b4221fe64cc381d71d38..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-33/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 12:47:18,207][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 12:48:13,606][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 12:46:33,052][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-22 12:46:33,491][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-34/.hydra/config.yaml b/outputs/2026-06-19/12-46-34/.hydra/config.yaml deleted file mode 100644 index ac3697b92d3aaa224d8690827ce376ca37c756e0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-34/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 4.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-34/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-34/.hydra/hydra.yaml deleted file mode 100644 index 628ca91751af2dabd9cc2aa500eeca56657ab30a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-34/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-34 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-34/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-34/.hydra/overrides.yaml deleted file mode 100644 index 9a9470fcb171b9de7af473034d3f72848dcf0e09..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-34/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ditlr4e5_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=4.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-34/main.log b/outputs/2026-06-19/12-46-34/main.log deleted file mode 100644 index 206b870550710ced25560a870f27080f40e8f2d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-34/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 12:47:21,694][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 12:48:13,571][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:17,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 12:46:33,051][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-22 12:46:34,530][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/12-46-48/.hydra/config.yaml b/outputs/2026-06-19/12-46-48/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-48/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-48/.hydra/hydra.yaml deleted file mode 100644 index 173a387de8dd533903fbdebf1c3330d2080992fd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-48/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-48/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-48/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-48/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-48/main.log b/outputs/2026-06-19/12-46-48/main.log deleted file mode 100644 index 4988beea5b0a23ad332dac9901c6ce26a5cbd7a9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-48/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 12:47:33,363][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 12:47:33,367][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 12:48:17,981][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:18,057][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/12-46-49/.hydra/config.yaml b/outputs/2026-06-19/12-46-49/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-49/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-49/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-49/.hydra/hydra.yaml deleted file mode 100644 index 0c7c669055405773602805740d14b6eb0a58173b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-49/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-49 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-49/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-49/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-49/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-49/main.log b/outputs/2026-06-19/12-46-49/main.log deleted file mode 100644 index e0916454afe22a44a4f7fd997be2bae2da0d040b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-49/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 12:47:33,356][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 12:47:33,382][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 12:48:18,049][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:18,091][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/12-46-50/.hydra/config.yaml b/outputs/2026-06-19/12-46-50/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-50/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-50/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-50/.hydra/hydra.yaml deleted file mode 100644 index 33c11df5201b886565adae260f759bf0c3bfaaaa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-50/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-50 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-50/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-50/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-50/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-50/main.log b/outputs/2026-06-19/12-46-50/main.log deleted file mode 100644 index 24bb205acc262eff0a2d112e17b92eb70fef3a2b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-50/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 12:47:34,003][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 12:48:16,940][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/12-46-51/.hydra/config.yaml b/outputs/2026-06-19/12-46-51/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-51/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-51/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-51/.hydra/hydra.yaml deleted file mode 100644 index 935b967d68e2f22601c22b5382ae35993cd04553..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-51/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-51 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-51/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-51/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-51/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-51/main.log b/outputs/2026-06-19/12-46-51/main.log deleted file mode 100644 index b9e3901b795c1ce3bdb9ef18780c3def2e3e4d57..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-51/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 12:47:33,379][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 12:48:18,067][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/12-46-52/.hydra/config.yaml b/outputs/2026-06-19/12-46-52/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-52/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-52/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-52/.hydra/hydra.yaml deleted file mode 100644 index 892ecd12491b6ead11e8f9fab969a46ae7259763..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-52/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-52 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-52/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-52/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-52/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-52/main.log b/outputs/2026-06-19/12-46-52/main.log deleted file mode 100644 index fb557cacc9aa042b65336685beee6372c30b1bb2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-52/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 12:47:33,385][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 12:48:16,960][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/12-46-53/.hydra/config.yaml b/outputs/2026-06-19/12-46-53/.hydra/config.yaml deleted file mode 100644 index 5f1434a7d8c57452d4e5434eff3f576e8af06352..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-53/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 0 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/12-46-53/.hydra/hydra.yaml b/outputs/2026-06-19/12-46-53/.hydra/hydra.yaml deleted file mode 100644 index f0d93acaaf9aae51aa27843e882db495f0ae1903..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-53/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=0 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=0,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/12-46-53 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/12-46-53/.hydra/overrides.yaml b/outputs/2026-06-19/12-46-53/.hydra/overrides.yaml deleted file mode 100644 index cd100937f0dd08e21d35eafd3b5f1531a2ba3251..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-53/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=0 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/12-46-53/main.log b/outputs/2026-06-19/12-46-53/main.log deleted file mode 100644 index e39079c6bb55ce4a8483b0c716a0abf685c1c984..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/12-46-53/main.log +++ /dev/null @@ -1,28 +0,0 @@ -[2026-06-19 12:47:30,487][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 12:47:30,651][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 12:47:30,651][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 12:47:30,652][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 12:47:30,652][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 12:47:30,652][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 12:47:33,380][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 12:47:39,783][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 12:47:55,031][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 12:48:17,958][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 12:48:19,575][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 12:48:20,760][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_anchor_d4r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 12:48:21,686][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-21 19:35:52,927][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=350000` reached. diff --git a/outputs/2026-06-19/13-50-53/.hydra/config.yaml b/outputs/2026-06-19/13-50-53/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-53/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-53/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-53/.hydra/hydra.yaml deleted file mode 100644 index 3e9b3acf6cf0f5e97994e4ad49571a0ec8debbf8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-53/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-53 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-53/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-53/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-53/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-53/main.log b/outputs/2026-06-19/13-50-53/main.log deleted file mode 100644 index 38d3566f3dce1c0e1ec87d763e1d21adf6149b5b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-53/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,736][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 diff --git a/outputs/2026-06-19/13-50-54/.hydra/config.yaml b/outputs/2026-06-19/13-50-54/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-54/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-54/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-54/.hydra/hydra.yaml deleted file mode 100644 index 4808628f916552b8b85d6b47b395b30469a90810..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-54/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-54 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-54/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-54/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-54/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-54/main.log b/outputs/2026-06-19/13-50-54/main.log deleted file mode 100644 index d11f414f6dccf5b0f5caefcfe9b24edda43ffcd1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-54/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,749][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 diff --git a/outputs/2026-06-19/13-50-55/.hydra/config.yaml b/outputs/2026-06-19/13-50-55/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-55/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-55/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-55/.hydra/hydra.yaml deleted file mode 100644 index a5aaaad5e172c4c5ee69fa79c43ffb3b3e6060e1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-55/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-55 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-55/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-55/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-55/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-55/main.log b/outputs/2026-06-19/13-50-55/main.log deleted file mode 100644 index 8979a4bf03e3a87518934b159a2ad5fa9ca31159..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-55/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,742][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 diff --git a/outputs/2026-06-19/13-50-56/.hydra/config.yaml b/outputs/2026-06-19/13-50-56/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-56/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-56/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-56/.hydra/hydra.yaml deleted file mode 100644 index f5a79ca50eb0f561b96a1eb1897a0bd390acc8db..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-56/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-56 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-56/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-56/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-56/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-56/main.log b/outputs/2026-06-19/13-50-56/main.log deleted file mode 100644 index a7c11afd2b4a1e364b1a1fad020b700a866c977e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-56/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,744][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 diff --git a/outputs/2026-06-19/13-50-57/.hydra/config.yaml b/outputs/2026-06-19/13-50-57/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-57/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-57/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-57/.hydra/hydra.yaml deleted file mode 100644 index cbe47ddf398e3578e162feee375e1b84b6c4766f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-57/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-57 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-57/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-57/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-57/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-57/main.log b/outputs/2026-06-19/13-50-57/main.log deleted file mode 100644 index fa72a7cec08fe579041828edefb8c55b0006adb6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-57/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,737][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 diff --git a/outputs/2026-06-19/13-50-58/.hydra/config.yaml b/outputs/2026-06-19/13-50-58/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-58/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-58/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-58/.hydra/hydra.yaml deleted file mode 100644 index 3f47b52b95efc2804bcfc0cfd45c514a620fa588..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-58/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-58 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-58/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-58/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-58/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-58/main.log b/outputs/2026-06-19/13-50-58/main.log deleted file mode 100644 index ce4c6f23e9ec2ae8a0f9ca77ab277b02010d567b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-58/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,765][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 diff --git a/outputs/2026-06-19/13-50-59/.hydra/config.yaml b/outputs/2026-06-19/13-50-59/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-59/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-50-59/.hydra/hydra.yaml b/outputs/2026-06-19/13-50-59/.hydra/hydra.yaml deleted file mode 100644 index b15331e950014b037dcfd17fc2915965b24d6896..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-59/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-50-59 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-50-59/.hydra/overrides.yaml b/outputs/2026-06-19/13-50-59/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-59/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-50-59/main.log b/outputs/2026-06-19/13-50-59/main.log deleted file mode 100644 index 1616ed9db0ebb14aaea7332a8ebe259a245530ee..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-50-59/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 13:51:44,744][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 diff --git a/outputs/2026-06-19/13-51-00/.hydra/config.yaml b/outputs/2026-06-19/13-51-00/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-51-00/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/13-51-00/.hydra/hydra.yaml b/outputs/2026-06-19/13-51-00/.hydra/hydra.yaml deleted file mode 100644 index 58e297d8d6e78f6c2ecdccf0c290bba928465b1d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-51-00/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/13-51-00 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/13-51-00/.hydra/overrides.yaml b/outputs/2026-06-19/13-51-00/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-51-00/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/13-51-00/main.log b/outputs/2026-06-19/13-51-00/main.log deleted file mode 100644 index c98d6fbd653d133639c0305db3dd39616efeccf0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/13-51-00/main.log +++ /dev/null @@ -1,14 +0,0 @@ -[2026-06-19 13:51:38,447][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 13:51:38,588][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 13:51:38,589][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 13:51:38,589][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 13:51:38,589][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 13:51:38,589][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 13:51:41,316][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 13:51:42,540][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 13:51:44,753][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 13:51:50,732][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/14-11-20/.hydra/config.yaml b/outputs/2026-06-19/14-11-20/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-20/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-20/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-20/.hydra/hydra.yaml deleted file mode 100644 index 7e1f5c54cf62d620f28d96e7e3cad6e5e360a34d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-20/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-20 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-20/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-20/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-20/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-20/main.log b/outputs/2026-06-19/14-11-20/main.log deleted file mode 100644 index 03b4bb1c191acfef8452948d96f11d9d785a1492..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-20/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 14:12:05,934][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 14:12:53,020][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/14-11-21/.hydra/config.yaml b/outputs/2026-06-19/14-11-21/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-21/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-21/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-21/.hydra/hydra.yaml deleted file mode 100644 index b777e83bee2e3b9e6868f71b6d2f20e85d8c8c9b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-21/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-21 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-21/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-21/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-21/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-21/main.log b/outputs/2026-06-19/14-11-21/main.log deleted file mode 100644 index ff10450f635d1c0817ce591bf417b56d8e76512c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-21/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 14:12:05,480][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 14:12:05,483][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 14:12:52,557][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:53,032][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/14-11-22/.hydra/config.yaml b/outputs/2026-06-19/14-11-22/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-22/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-22/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-22/.hydra/hydra.yaml deleted file mode 100644 index 778a8459e32baae885eae0484a25caee667eeb43..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-22/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-22 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-22/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-22/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-22/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-22/main.log b/outputs/2026-06-19/14-11-22/main.log deleted file mode 100644 index 03b3d50ce6910bb0365c8bc54c8c9fe1d02496e9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-22/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 14:12:05,578][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 14:12:53,039][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/14-11-23/.hydra/config.yaml b/outputs/2026-06-19/14-11-23/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-23/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-23/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-23/.hydra/hydra.yaml deleted file mode 100644 index 42d02e8c640e40854cda299154f9da25ac0f211f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-23/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-23 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-23/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-23/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-23/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-23/main.log b/outputs/2026-06-19/14-11-23/main.log deleted file mode 100644 index a25b3dd59f6b0b5ba76592a144d0b1f1ac6fedc4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-23/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 14:12:05,473][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 14:12:53,039][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/14-11-24/.hydra/config.yaml b/outputs/2026-06-19/14-11-24/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-24/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-24/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-24/.hydra/hydra.yaml deleted file mode 100644 index 5ae5d18eb32951e955a315195809838b07929e0c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-24/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-24 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-24/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-24/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-24/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-24/main.log b/outputs/2026-06-19/14-11-24/main.log deleted file mode 100644 index 6a1ea010b2c70c74c69403671941a6dc16fdd3ca..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-24/main.log +++ /dev/null @@ -1,28 +0,0 @@ -[2026-06-19 14:12:02,642][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 14:12:02,779][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 14:12:02,779][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 14:12:02,779][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 14:12:02,779][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 14:12:02,780][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 14:12:05,492][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 14:12:11,974][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 14:12:27,493][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 14:12:52,555][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:56,384][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 14:12:57,392][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-21 12:57:55,245][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=350000` reached. diff --git a/outputs/2026-06-19/14-11-25/.hydra/config.yaml b/outputs/2026-06-19/14-11-25/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-25/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-25/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-25/.hydra/hydra.yaml deleted file mode 100644 index 08379165b429c84666d34bd56c13e30ccd818074..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-25/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-25 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-25/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-25/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-25/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-25/main.log b/outputs/2026-06-19/14-11-25/main.log deleted file mode 100644 index a0f8fffe8ba537d974437c806d4937e3fa26c35f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-25/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 14:12:05,488][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 14:12:53,037][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/14-11-26/.hydra/config.yaml b/outputs/2026-06-19/14-11-26/.hydra/config.yaml deleted file mode 100644 index 00a4819247bb946a4bde3e323b2ace07435b877f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-26/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/14-11-26/.hydra/hydra.yaml b/outputs/2026-06-19/14-11-26/.hydra/hydra.yaml deleted file mode 100644 index 39cdcb95392f79499a6493385a86996af1ea9fcd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-26/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/14-11-26 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/14-11-26/.hydra/overrides.yaml b/outputs/2026-06-19/14-11-26/.hydra/overrides.yaml deleted file mode 100644 index c327e3e8696cb770e644c8cdb4761d5660506390..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-26/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_dynamic_a2r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/14-11-26/main.log b/outputs/2026-06-19/14-11-26/main.log deleted file mode 100644 index d2102f455c136cc9e9e6e0cd69bf2e0b3a8e3f2c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/14-11-26/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 14:12:05,455][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 14:12:53,027][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 14:12:58,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/15-18-30/.hydra/config.yaml b/outputs/2026-06-19/15-18-30/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-30/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-18-30/.hydra/hydra.yaml b/outputs/2026-06-19/15-18-30/.hydra/hydra.yaml deleted file mode 100644 index a716fff2693ad1ea5c8c261e929782881ab5f49b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-30/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-18-30 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-18-30/.hydra/overrides.yaml b/outputs/2026-06-19/15-18-30/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-30/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-18-30/main.log b/outputs/2026-06-19/15-18-30/main.log deleted file mode 100644 index b3f8e49514325549894b25df199a624c1cbe6e9a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-30/main.log +++ /dev/null @@ -1,16 +0,0 @@ -[2026-06-19 15:19:03,194][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 15:19:03,350][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 15:19:03,350][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 15:19:03,350][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 15:19:03,350][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 15:19:03,351][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 15:19:03,467][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 15:19:03,590][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 15:19:05,102][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 15:19:05,208][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 15:19:05,668][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 15:19:11,405][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/15-18-31/.hydra/config.yaml b/outputs/2026-06-19/15-18-31/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-31/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-18-31/.hydra/hydra.yaml b/outputs/2026-06-19/15-18-31/.hydra/hydra.yaml deleted file mode 100644 index 856e66b5c32a8b969114891a46b6282ac21f286c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-31/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-18-31 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-18-31/.hydra/overrides.yaml b/outputs/2026-06-19/15-18-31/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-31/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-18-31/main.log b/outputs/2026-06-19/15-18-31/main.log deleted file mode 100644 index 3405f7f5639dcfc2b35029b5f69affbb062aede9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-31/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 15:19:05,465][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 15:19:05,952][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 diff --git a/outputs/2026-06-19/15-18-33/.hydra/config.yaml b/outputs/2026-06-19/15-18-33/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-33/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-18-33/.hydra/hydra.yaml b/outputs/2026-06-19/15-18-33/.hydra/hydra.yaml deleted file mode 100644 index 53bc79e1c65849d08f14ac8d62a3ade72588874b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-33/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-18-33 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-18-33/.hydra/overrides.yaml b/outputs/2026-06-19/15-18-33/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-33/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-18-33/main.log b/outputs/2026-06-19/15-18-33/main.log deleted file mode 100644 index 53b5bea23d32cd8ea53699461ef468a73091884f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-33/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 15:19:05,166][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 diff --git a/outputs/2026-06-19/15-18-34/.hydra/config.yaml b/outputs/2026-06-19/15-18-34/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-34/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-18-34/.hydra/hydra.yaml b/outputs/2026-06-19/15-18-34/.hydra/hydra.yaml deleted file mode 100644 index 9899b6658c221bee3e0e49bc0413dbb0e2780102..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-34/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-18-34 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-18-34/.hydra/overrides.yaml b/outputs/2026-06-19/15-18-34/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-34/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-18-34/main.log b/outputs/2026-06-19/15-18-34/main.log deleted file mode 100644 index 8589897da675e2392db7d728f2a6c96c8fe44949..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-34/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 15:19:05,048][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 diff --git a/outputs/2026-06-19/15-18-35/.hydra/config.yaml b/outputs/2026-06-19/15-18-35/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-35/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-18-35/.hydra/hydra.yaml b/outputs/2026-06-19/15-18-35/.hydra/hydra.yaml deleted file mode 100644 index 194ecf2cb2a885eb796a679560ed5a1544bf3c51..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-35/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-18-35 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-18-35/.hydra/overrides.yaml b/outputs/2026-06-19/15-18-35/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-35/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-18-35/main.log b/outputs/2026-06-19/15-18-35/main.log deleted file mode 100644 index 45cf754d1e6cf633251893aa63e3a47a1a557cc2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-18-35/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 15:19:06,637][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 diff --git a/outputs/2026-06-19/15-19-45/.hydra/config.yaml b/outputs/2026-06-19/15-19-45/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-45/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-19-45/.hydra/hydra.yaml b/outputs/2026-06-19/15-19-45/.hydra/hydra.yaml deleted file mode 100644 index be048b5b4af5419ae5dd62de51a210573bcfa36d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-45/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-19-45 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-19-45/.hydra/overrides.yaml b/outputs/2026-06-19/15-19-45/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-45/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-19-45/main.log b/outputs/2026-06-19/15-19-45/main.log deleted file mode 100644 index 5e2c710d874aee5e2025e2e1b8d5cbc46589e116..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-45/main.log +++ /dev/null @@ -1,17 +0,0 @@ -[2026-06-19 15:20:16,054][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 15:20:16,200][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 15:20:16,200][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 15:20:16,200][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 15:20:16,201][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 15:20:16,201][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 15:20:16,302][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 15:20:16,415][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 15:20:18,103][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 15:20:18,105][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 15:20:18,427][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 15:20:18,606][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 15:20:23,766][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/15-19-46/.hydra/config.yaml b/outputs/2026-06-19/15-19-46/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-46/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-19-46/.hydra/hydra.yaml b/outputs/2026-06-19/15-19-46/.hydra/hydra.yaml deleted file mode 100644 index fe81399a285d8cc34b86514c6583d811b7216129..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-46/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-19-46 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-19-46/.hydra/overrides.yaml b/outputs/2026-06-19/15-19-46/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-46/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-19-46/main.log b/outputs/2026-06-19/15-19-46/main.log deleted file mode 100644 index 71ffbfb6275df57a213495ee837e2ebaaa49bd9c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-46/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 15:20:18,219][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 15:20:19,496][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 diff --git a/outputs/2026-06-19/15-19-47/.hydra/config.yaml b/outputs/2026-06-19/15-19-47/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-47/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-19-47/.hydra/hydra.yaml b/outputs/2026-06-19/15-19-47/.hydra/hydra.yaml deleted file mode 100644 index f4ba125198c3f7deea52e33ed1a7debe9b93c409..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-47/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-19-47 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-19-47/.hydra/overrides.yaml b/outputs/2026-06-19/15-19-47/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-47/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-19-47/main.log b/outputs/2026-06-19/15-19-47/main.log deleted file mode 100644 index 64b4ecda91d3b00f121b54fd626a2b5f78fb414a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-47/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 15:20:18,869][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 diff --git a/outputs/2026-06-19/15-19-48/.hydra/config.yaml b/outputs/2026-06-19/15-19-48/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-19-48/.hydra/hydra.yaml b/outputs/2026-06-19/15-19-48/.hydra/hydra.yaml deleted file mode 100644 index b551fa5f59fdd4568a97763cd890429afdc27c59..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-48/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-19-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-19-48/.hydra/overrides.yaml b/outputs/2026-06-19/15-19-48/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-48/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/15-19-48/main.log b/outputs/2026-06-19/15-19-48/main.log deleted file mode 100644 index c17189f630c17c341bf3a86b340472bd6870d018..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-19-48/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 15:20:18,197][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 diff --git a/outputs/2026-06-19/15-54-21/.hydra/config.yaml b/outputs/2026-06-19/15-54-21/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-21/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-21/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-21/.hydra/hydra.yaml deleted file mode 100644 index fa81110577f7cd47e7a41cb3dba4c5887d64175b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-21/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-21 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-21/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-21/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-21/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-21/main.log b/outputs/2026-06-19/15-54-21/main.log deleted file mode 100644 index 9917a4a2e3ed60da32dd0a7713fdad8cf873c62f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-21/main.log +++ /dev/null @@ -1,15 +0,0 @@ -[2026-06-19 15:55:07,715][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 15:55:07,746][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 15:55:11,454][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 15:55:55,569][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:55:55,608][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:55:55,645][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:56:00,119][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 15:56:00,120][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 15:56:00,120][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-22 15:54:35,175][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-22 15:54:35,175][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-22 15:54:35,173][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-22 15:54:35,176][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/15-54-22/.hydra/config.yaml b/outputs/2026-06-19/15-54-22/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-22/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-22/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-22/.hydra/hydra.yaml deleted file mode 100644 index 82f4764e07490c8f867af37fd8a518e97fcf16ac..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-22/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-22 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-22/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-22/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-22/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-22/main.log b/outputs/2026-06-19/15-54-22/main.log deleted file mode 100644 index c76ae0bb36dae1a643d1b21aa45918f0996e13ce..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-22/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 15:55:08,475][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 15:55:55,589][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:56:00,120][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,175][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-22 15:54:35,176][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/15-54-23/.hydra/config.yaml b/outputs/2026-06-19/15-54-23/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-23/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-23/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-23/.hydra/hydra.yaml deleted file mode 100644 index 26dff8841a5c25597a805199d2fd214f0dafcd14..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-23/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-23 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-23/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-23/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-23/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-23/main.log b/outputs/2026-06-19/15-54-23/main.log deleted file mode 100644 index 0cc5a3c70a5b469c6ca9187a33a6835b68b668cd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-23/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 15:55:07,944][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 15:55:55,645][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:56:00,119][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-22 15:54:35,175][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/15-54-24/.hydra/config.yaml b/outputs/2026-06-19/15-54-24/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-24/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-24/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-24/.hydra/hydra.yaml deleted file mode 100644 index 2b16130094e0fe6e8f315e029aa8175765f4b44b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-24/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-24 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-24/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-24/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-24/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-24/main.log b/outputs/2026-06-19/15-54-24/main.log deleted file mode 100644 index d2980ecf73db3809ae4704bf0707451cc31e65d5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-24/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 15:55:07,720][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 15:55:55,632][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:56:00,119][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-22 15:54:35,176][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/15-54-25/.hydra/config.yaml b/outputs/2026-06-19/15-54-25/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-25/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-25/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-25/.hydra/hydra.yaml deleted file mode 100644 index 5fdd079b000b6da3736ca12d6a0fe86504fbf947..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-25/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-25 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-25/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-25/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-25/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-25/main.log b/outputs/2026-06-19/15-54-25/main.log deleted file mode 100644 index 39c626b22b2c9126f0f544535ff4b4ecaa394845..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-25/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-19 15:55:07,727][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 15:55:55,129][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:56:00,120][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-22 15:54:35,176][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/15-54-26/.hydra/config.yaml b/outputs/2026-06-19/15-54-26/.hydra/config.yaml deleted file mode 100644 index 9869f51c161adbbd6d3c059d1e67ed8908232dc6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-26/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/15-54-26/.hydra/hydra.yaml b/outputs/2026-06-19/15-54-26/.hydra/hydra.yaml deleted file mode 100644 index a4fe6a82310588eaaa21f6d606f0eeeb651aa402..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-26/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/15-54-26 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/15-54-26/.hydra/overrides.yaml b/outputs/2026-06-19/15-54-26/.hydra/overrides.yaml deleted file mode 100644 index 5e998fce7e79fdbf9bdb2057d2f99d7858591ce3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-26/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/15-54-26/main.log b/outputs/2026-06-19/15-54-26/main.log deleted file mode 100644 index 5b69e4f8b61e484e27b35b9bccc7ba23adb83d03..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/15-54-26/main.log +++ /dev/null @@ -1,29 +0,0 @@ -[2026-06-19 15:55:04,888][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 15:55:06,098][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 15:55:06,098][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 15:55:06,098][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 15:55:06,098][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 15:55:06,099][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 15:55:08,275][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 15:55:15,852][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 15:55:29,583][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 15:55:55,114][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 15:55:57,124][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 15:55:58,057][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8a100_bs8_global64_350k/checkpoints/last.ckpt -[2026-06-19 15:56:00,120][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 15:54:35,174][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-22 15:54:35,176][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/18-56-25/.hydra/config.yaml b/outputs/2026-06-19/18-56-25/.hydra/config.yaml deleted file mode 100644 index b47e3332559c53ed999c275a717b07d1982311f8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-25/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/18-56-25/.hydra/hydra.yaml b/outputs/2026-06-19/18-56-25/.hydra/hydra.yaml deleted file mode 100644 index 6319433939100c419c3ca27b905ebfa878fc72d1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-25/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=[3,7,11,15] - - ++algorithm.dememwm.injection.memory_hidden_size=512 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=512,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=[3,7,11,15],algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/18-56-25 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/18-56-25/.hydra/overrides.yaml b/outputs/2026-06-19/18-56-25/.hydra/overrides.yaml deleted file mode 100644 index 8ee3288e706e04827ca476ead0592fea9c288ffa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-25/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=[3,7,11,15] -- ++algorithm.dememwm.injection.memory_hidden_size=512 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/18-56-25/main.log b/outputs/2026-06-19/18-56-25/main.log deleted file mode 100644 index dbc487c7e1fde877599da4f92c945c87ea609e49..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-25/main.log +++ /dev/null @@ -1,20 +0,0 @@ -[2026-06-19 18:57:06,921][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 18:57:07,170][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 18:57:07,278][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 18:57:07,358][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 18:57:46,640][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,229][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,231][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,287][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:04:55,613][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-19 19:04:55,615][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/18-56-26/.hydra/config.yaml b/outputs/2026-06-19/18-56-26/.hydra/config.yaml deleted file mode 100644 index b47e3332559c53ed999c275a717b07d1982311f8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-26/.hydra/config.yaml +++ /dev/null @@ -1,231 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - customized_validation: true - wo_updown: false - angle_range: 180 - pos_range: 8 -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_memory_dit - memory_token_cross_attention: true - memory_cross_attn_layers: - - 3 - - 7 - - 11 - - 15 - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - training_stage: stage_1 - curriculum: - enabled: true - full_stage_start_step: 20000 - freeze_vae: true - dit_freeze: - enabled: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - spatial_kernel_size: 3 - anchor: - enabled: true - anchor_indices: - - 0 - - 1 - - 2 - - 3 - allow_generated_as_anchor: false - diverse_selection: true - compress: - downsample_ratio: 2 - dynamic: - enabled: true - recent_frames: 4 - conv_kernel_t: 3 - conv_stride_t: 2 - compress: - downsample_ratio: 2 - revisit: - enabled: true - deterministic_pose_retrieval: true - fov_overlap_threshold: 0.6 - plucker_weight: 0.1 - max_frames: 2 - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - compress: - downsample_ratio: 2 - eval_ablation: - enabled: false - branch: A_plus_D_plus_R_normal - generated_history_proxy: - enabled: true - start_step: 40000 - ramp_steps: 40000 - max_prob: 0.25 - noise_std: 0.25 - dropout_prob: 0.0 - injection: - memory_hidden_size: 512 - cache: - enabled: true - device: cpu - keep_raw_latents: all - keep_compressed_records: true - keep_prefix_anchors: true - eviction_policy: none - no_evict: true - clear_between_videos: true - max_records: null - on_capacity_exceeded: warn - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 4 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/18-56-26/.hydra/hydra.yaml b/outputs/2026-06-19/18-56-26/.hydra/hydra.yaml deleted file mode 100644 index cb40de80e346caa3e9f7df77b383a6642f9e8f3c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-26/.hydra/hydra.yaml +++ /dev/null @@ -1,232 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_memory_dit - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - +dataset.customized_validation=true - - +dataset.wo_updown=false - - +dataset.angle_range=180 - - +dataset.pos_range=8 - - ++algorithm.n_tokens=4 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.memory_cross_attn_layers=[3,7,11,15] - - ++algorithm.dememwm.injection.memory_hidden_size=512 - - ++algorithm.dememwm.generated_history_proxy.enabled=true - - ++algorithm.dememwm.generated_history_proxy.start_step=40000 - - ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 - - ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 - - ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 - - ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 - - ++algorithm.dememwm.anchor.enabled=true - - ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] - - ++algorithm.dememwm.anchor.diverse_selection=true - - ++algorithm.dememwm.anchor.compress.downsample_ratio=2 - - ++algorithm.dememwm.anchor.allow_generated_as_anchor=false - - ++algorithm.dememwm.dynamic.enabled=true - - ++algorithm.dememwm.dynamic.recent_frames=4 - - ++algorithm.dememwm.dynamic.conv_stride_t=2 - - ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 - - ++algorithm.dememwm.revisit.enabled=true - - ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true - - ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.max_frames=2 - - ++algorithm.dememwm.revisit.compress.downsample_ratio=2 - - ++algorithm.dememwm.cache.enabled=true - - ++algorithm.dememwm.cache.device=cpu - - ++algorithm.dememwm.cache.keep_raw_latents=all - - ++algorithm.dememwm.cache.keep_compressed_records=true - - ++algorithm.dememwm.cache.eviction_policy=none - - ++algorithm.dememwm.cache.no_evict=true - - ++algorithm.dememwm.cache.clear_between_videos=true - - ++algorithm.dememwm.cache.max_records=null - - ++algorithm.dememwm.cache.on_capacity_exceeded=warn - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=20000 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.dit_freeze.enabled=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.anchor.allow_generated_as_anchor=false,++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3],++algorithm.dememwm.anchor.compress.downsample_ratio=2,++algorithm.dememwm.anchor.diverse_selection=true,++algorithm.dememwm.anchor.enabled=true,++algorithm.dememwm.cache.clear_between_videos=true,++algorithm.dememwm.cache.device=cpu,++algorithm.dememwm.cache.enabled=true,++algorithm.dememwm.cache.eviction_policy=none,++algorithm.dememwm.cache.keep_compressed_records=true,++algorithm.dememwm.cache.keep_raw_latents=all,++algorithm.dememwm.cache.max_records=null,++algorithm.dememwm.cache.no_evict=true,++algorithm.dememwm.cache.on_capacity_exceeded=warn,++algorithm.dememwm.curriculum.dit_freeze.enabled=true,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=20000,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.dynamic.compress.downsample_ratio=2,++algorithm.dememwm.dynamic.conv_stride_t=2,++algorithm.dememwm.dynamic.enabled=true,++algorithm.dememwm.dynamic.recent_frames=4,++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0,++algorithm.dememwm.generated_history_proxy.enabled=true,++algorithm.dememwm.generated_history_proxy.max_prob=0.25,++algorithm.dememwm.generated_history_proxy.noise_std=0.25,++algorithm.dememwm.generated_history_proxy.ramp_steps=40000,++algorithm.dememwm.generated_history_proxy.start_step=40000,++algorithm.dememwm.injection.memory_hidden_size=512,++algorithm.dememwm.revisit.compress.downsample_ratio=2,++algorithm.dememwm.revisit.deterministic_pose_retrieval=true,++algorithm.dememwm.revisit.enabled=true,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_overlap_threshold=0.60,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.max_frames=2,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=4,+customized_load=true,+dataset.angle_range=180,+dataset.customized_validation=true,+dataset.n_frames_valid=500,+dataset.pos_range=8,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.memory_cross_attn_layers=[3,7,11,15],algorithm.x_shape=[16,18,32],algorithm=dememwm_memory_dit,auto_resume=true,dataset.n_frames=1000,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/18-56-26 - choices: - cluster: null - algorithm: dememwm_memory_dit - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/18-56-26/.hydra/overrides.yaml b/outputs/2026-06-19/18-56-26/.hydra/overrides.yaml deleted file mode 100644 index 8ee3288e706e04827ca476ead0592fea9c288ffa..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-26/.hydra/overrides.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- +name=train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_memory_dit -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- +dataset.customized_validation=true -- +dataset.wo_updown=false -- +dataset.angle_range=180 -- +dataset.pos_range=8 -- ++algorithm.n_tokens=4 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.memory_cross_attn_layers=[3,7,11,15] -- ++algorithm.dememwm.injection.memory_hidden_size=512 -- ++algorithm.dememwm.generated_history_proxy.enabled=true -- ++algorithm.dememwm.generated_history_proxy.start_step=40000 -- ++algorithm.dememwm.generated_history_proxy.ramp_steps=40000 -- ++algorithm.dememwm.generated_history_proxy.max_prob=0.25 -- ++algorithm.dememwm.generated_history_proxy.noise_std=0.25 -- ++algorithm.dememwm.generated_history_proxy.dropout_prob=0.0 -- ++algorithm.dememwm.anchor.enabled=true -- ++algorithm.dememwm.anchor.anchor_indices=[0,1,2,3] -- ++algorithm.dememwm.anchor.diverse_selection=true -- ++algorithm.dememwm.anchor.compress.downsample_ratio=2 -- ++algorithm.dememwm.anchor.allow_generated_as_anchor=false -- ++algorithm.dememwm.dynamic.enabled=true -- ++algorithm.dememwm.dynamic.recent_frames=4 -- ++algorithm.dememwm.dynamic.conv_stride_t=2 -- ++algorithm.dememwm.dynamic.compress.downsample_ratio=2 -- ++algorithm.dememwm.revisit.enabled=true -- ++algorithm.dememwm.revisit.deterministic_pose_retrieval=true -- ++algorithm.dememwm.revisit.fov_overlap_threshold=0.60 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.max_frames=2 -- ++algorithm.dememwm.revisit.compress.downsample_ratio=2 -- ++algorithm.dememwm.cache.enabled=true -- ++algorithm.dememwm.cache.device=cpu -- ++algorithm.dememwm.cache.keep_raw_latents=all -- ++algorithm.dememwm.cache.keep_compressed_records=true -- ++algorithm.dememwm.cache.eviction_policy=none -- ++algorithm.dememwm.cache.no_evict=true -- ++algorithm.dememwm.cache.clear_between_videos=true -- ++algorithm.dememwm.cache.max_records=null -- ++algorithm.dememwm.cache.on_capacity_exceeded=warn -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=20000 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.dit_freeze.enabled=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-19/18-56-26/main.log b/outputs/2026-06-19/18-56-26/main.log deleted file mode 100644 index 4df50e7e80675c88f553b0c00b1ff48df43f9df2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/18-56-26/main.log +++ /dev/null @@ -1,48 +0,0 @@ -[2026-06-19 18:57:04,868][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 18:57:05,006][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 18:57:05,006][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 18:57:05,006][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 18:57:05,006][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 18:57:05,007][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 18:57:06,925][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 18:57:06,926][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 18:57:06,942][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 18:57:06,949][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 18:57:12,511][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 18:57:30,579][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/checkpoints/last.ckpt -[2026-06-19 18:57:47,228][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,230][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,230][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:47,344][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 18:57:48,995][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params -------------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 681 M -1 | dememwm_dynamic_compressor | CausalConv3DDynamicCompressor | 887 K -2 | dememwm_anchor_proj | SpatialConv2DMemoryProjector | 296 K -3 | dememwm_revisit_proj | SpatialConv2DMemoryProjector | 296 K -4 | dememwm_revisit_gate | RevisitRawGate | 4 -5 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -6 | vae | AutoencoderKL | 229 M -------------------------------------------------------------------------------------- -682 M Trainable params -231 M Non-trainable params -914 M Total params -3,658.543 Total estimated model params size (MB) -[2026-06-19 18:57:49,674][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_separate_memory_gatefix_4layers512_valid500_berzelius_8a100_bs16_global128_350k/checkpoints/last.ckpt -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,222][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 18:57:52,223][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-19 19:04:55,613][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-19 19:04:55,613][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-19 19:04:55,614][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-19/19-05-08/.hydra/config.yaml b/outputs/2026-06-19/19-05-08/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-08/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-08/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-08/.hydra/hydra.yaml deleted file mode 100644 index 35ccf34cb07324f71c92458d94078f6924679b77..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-08/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-08 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-08/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-08/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-08/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-08/main.log b/outputs/2026-06-19/19-05-08/main.log deleted file mode 100644 index 11c24081dba1be91ef7588f0dc994871d422f054..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-08/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:06:02,006][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 diff --git a/outputs/2026-06-19/19-05-09/.hydra/config.yaml b/outputs/2026-06-19/19-05-09/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-09/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-09/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-09/.hydra/hydra.yaml deleted file mode 100644 index 21b6a244a54a15710c68ecb8199e8dc7aebf189f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-09/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-09 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-09/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-09/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-09/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-09/main.log b/outputs/2026-06-19/19-05-09/main.log deleted file mode 100644 index a90452ca5bc0e971c6383abd05fb68f13a820e77..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-09/main.log +++ /dev/null @@ -1,14 +0,0 @@ -[2026-06-19 19:05:55,848][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 19:05:56,008][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 19:05:56,009][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 19:05:56,009][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 19:05:56,009][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 19:05:56,009][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 19:05:58,400][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:05:59,667][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:06:02,045][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 19:06:09,287][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/19-05-12/.hydra/config.yaml b/outputs/2026-06-19/19-05-12/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-12/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-12/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-12/.hydra/hydra.yaml deleted file mode 100644 index 2b6d5146597d7abef75cfc44d2c51863939cfb36..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-12/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-12 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-12/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-12/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-12/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-12/main.log b/outputs/2026-06-19/19-05-12/main.log deleted file mode 100644 index d7ed6ed3c73a349a204b0a811e915d68481732a0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-12/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 19:06:02,001][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 19:06:02,006][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 diff --git a/outputs/2026-06-19/19-05-13/.hydra/config.yaml b/outputs/2026-06-19/19-05-13/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-13/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-13/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-13/.hydra/hydra.yaml deleted file mode 100644 index 54e36110b69b5a645f9ec88c5da7c844b8e4f4d7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-13/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-13 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-13/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-13/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-13/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-13/main.log b/outputs/2026-06-19/19-05-13/main.log deleted file mode 100644 index 566749444d3baf62cf5b95f67c7d15f085d71346..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-13/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 19:06:02,002][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 19:06:02,031][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 diff --git a/outputs/2026-06-19/19-05-14/.hydra/config.yaml b/outputs/2026-06-19/19-05-14/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-14/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-14/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-14/.hydra/hydra.yaml deleted file mode 100644 index 4f1b62e6b712abdf2eb4fd160d9ed74e82e1eb72..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-14/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-14 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-14/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-14/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-14/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-14/main.log b/outputs/2026-06-19/19-05-14/main.log deleted file mode 100644 index efb1add8f763818cb29235a2d8425d1e9ca5f68b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-14/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:06:02,012][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 diff --git a/outputs/2026-06-19/19-05-15/.hydra/config.yaml b/outputs/2026-06-19/19-05-15/.hydra/config.yaml deleted file mode 100644 index c59a632cf78c535166e69ca7f9d7cc18ae1e3f3b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-15/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-05-15/.hydra/hydra.yaml b/outputs/2026-06-19/19-05-15/.hydra/hydra.yaml deleted file mode 100644 index c8fa99af23acfa34c271ae7a716546d2c85567df..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-15/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-05-15 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-05-15/.hydra/overrides.yaml b/outputs/2026-06-19/19-05-15/.hydra/overrides.yaml deleted file mode 100644 index 7dc9321e7110bcf1e972bf4ff4e64f393e83fa21..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-15/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-05-15/main.log b/outputs/2026-06-19/19-05-15/main.log deleted file mode 100644 index a0e87af8f98e6e6afaf58b74cfe0f0b18a6e8909..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-05-15/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:06:02,001][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 diff --git a/outputs/2026-06-19/19-06-42/.hydra/config.yaml b/outputs/2026-06-19/19-06-42/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-42/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-42/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-42/.hydra/hydra.yaml deleted file mode 100644 index 855c5c92bc6723d8340ff86ea468c4b9dd8d68d6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-42/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-42 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-42/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-42/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-42/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-42/main.log b/outputs/2026-06-19/19-06-42/main.log deleted file mode 100644 index 65d98268ccd1718bfb4637845a1839165fa01541..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-42/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:07:19,033][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 diff --git a/outputs/2026-06-19/19-06-43/.hydra/config.yaml b/outputs/2026-06-19/19-06-43/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-43/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-43/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-43/.hydra/hydra.yaml deleted file mode 100644 index 2834521d34683d9089c24181b4a1ea53041686f2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-43/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-43 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-43/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-43/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-43/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-43/main.log b/outputs/2026-06-19/19-06-43/main.log deleted file mode 100644 index c273f1cf389c6154046de7f636e8a2b778ab5db5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-43/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 19:07:18,962][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 19:07:19,021][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 diff --git a/outputs/2026-06-19/19-06-44/.hydra/config.yaml b/outputs/2026-06-19/19-06-44/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-44/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-44/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-44/.hydra/hydra.yaml deleted file mode 100644 index 9fbd52f1796a656f8f22b5150c44e17b753d5d1b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-44/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-44 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-44/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-44/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-44/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-44/main.log b/outputs/2026-06-19/19-06-44/main.log deleted file mode 100644 index ecda8bbb07aaeedc8f552d47a8d4c6ee4106908f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-44/main.log +++ /dev/null @@ -1,14 +0,0 @@ -[2026-06-19 19:07:14,036][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 19:07:14,176][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 19:07:14,177][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 19:07:14,177][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 19:07:14,177][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 19:07:14,177][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 19:07:16,054][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:07:17,372][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:07:18,994][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 19:07:25,541][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/19-06-45/.hydra/config.yaml b/outputs/2026-06-19/19-06-45/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-45/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-45/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-45/.hydra/hydra.yaml deleted file mode 100644 index ab52de1e296a483ae3adfdc70c72f9075638c8b2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-45/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-45 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-45/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-45/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-45/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-45/main.log b/outputs/2026-06-19/19-06-45/main.log deleted file mode 100644 index 63b54aa679fb57938001369b4f0b0c5fba13c09c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-45/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:07:18,963][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 diff --git a/outputs/2026-06-19/19-06-46/.hydra/config.yaml b/outputs/2026-06-19/19-06-46/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-46/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-46/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-46/.hydra/hydra.yaml deleted file mode 100644 index 57e81a930efd6a868e26732080eb69b682da7a58..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-46/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-46 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-46/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-46/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-46/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-46/main.log b/outputs/2026-06-19/19-06-46/main.log deleted file mode 100644 index 7bb7de78b1e7da924c1e6068e459dbd8d113797d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-46/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:07:18,972][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 diff --git a/outputs/2026-06-19/19-06-47/.hydra/config.yaml b/outputs/2026-06-19/19-06-47/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-47/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-47/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-47/.hydra/hydra.yaml deleted file mode 100644 index 945b02f52bd8f707df8247c3e001e89a4fd105e2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-47/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-47 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-47/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-47/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-47/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-47/main.log b/outputs/2026-06-19/19-06-47/main.log deleted file mode 100644 index 6716674bc4ee5ed76b5b02d12ebbb150adb79974..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-47/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:07:18,958][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 diff --git a/outputs/2026-06-19/19-06-48/.hydra/config.yaml b/outputs/2026-06-19/19-06-48/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-06-48/.hydra/hydra.yaml b/outputs/2026-06-19/19-06-48/.hydra/hydra.yaml deleted file mode 100644 index b10ca39e12de57c870f44971a0d42c2d07d0bd0d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-48/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-06-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-06-48/.hydra/overrides.yaml b/outputs/2026-06-19/19-06-48/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-48/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-06-48/main.log b/outputs/2026-06-19/19-06-48/main.log deleted file mode 100644 index efedf29d24756eb917018c4f92017e1e92af1c1c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-06-48/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:07:18,993][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 diff --git a/outputs/2026-06-19/19-07-41/.hydra/config.yaml b/outputs/2026-06-19/19-07-41/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-41/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-07-41/.hydra/hydra.yaml b/outputs/2026-06-19/19-07-41/.hydra/hydra.yaml deleted file mode 100644 index e534c6114b40af3363b58e2895e9b81c62e41af6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-41/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-07-41 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-07-41/.hydra/overrides.yaml b/outputs/2026-06-19/19-07-41/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-41/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-07-41/main.log b/outputs/2026-06-19/19-07-41/main.log deleted file mode 100644 index a705e6df481d9f4c13201aa3672d4a2bb22d5560..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-41/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:08:17,014][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 diff --git a/outputs/2026-06-19/19-07-42/.hydra/config.yaml b/outputs/2026-06-19/19-07-42/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-42/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-07-42/.hydra/hydra.yaml b/outputs/2026-06-19/19-07-42/.hydra/hydra.yaml deleted file mode 100644 index 7037f8f2eda344b1870b3456a71db4678ae2d21f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-42/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-07-42 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-07-42/.hydra/overrides.yaml b/outputs/2026-06-19/19-07-42/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-42/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-07-42/main.log b/outputs/2026-06-19/19-07-42/main.log deleted file mode 100644 index 25856a619345bdad65760ea9e25c0f0d4059a957..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-42/main.log +++ /dev/null @@ -1,2 +0,0 @@ -[2026-06-19 19:08:15,904][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 19:08:16,370][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 diff --git a/outputs/2026-06-19/19-07-43/.hydra/config.yaml b/outputs/2026-06-19/19-07-43/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-43/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-07-43/.hydra/hydra.yaml b/outputs/2026-06-19/19-07-43/.hydra/hydra.yaml deleted file mode 100644 index d1049a00b73ab5c5312c440aa5863bbe8ebfbfff..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-43/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-07-43 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-07-43/.hydra/overrides.yaml b/outputs/2026-06-19/19-07-43/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-43/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-07-43/main.log b/outputs/2026-06-19/19-07-43/main.log deleted file mode 100644 index 60e014fca511b23b2b9b70b050bcfcd3e261b79d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-43/main.log +++ /dev/null @@ -1,17 +0,0 @@ -[2026-06-19 19:08:14,332][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 19:08:15,193][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 19:08:15,193][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 19:08:15,193][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 19:08:15,193][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 19:08:15,194][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 19:08:15,299][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:08:15,397][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:08:16,134][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 19:08:16,172][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 19:08:16,867][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 19:08:17,015][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 19:08:22,716][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - diff --git a/outputs/2026-06-19/19-07-44/.hydra/config.yaml b/outputs/2026-06-19/19-07-44/.hydra/config.yaml deleted file mode 100644 index 123262f53d3a02bc01b30dd13312424c4c28d9c6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-44/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: online -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-07-44/.hydra/hydra.yaml b/outputs/2026-06-19/19-07-44/.hydra/hydra.yaml deleted file mode 100644 index c73577a85e32c4cc1fcd6f16a896300059d02b67..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-44/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - wandb.mode=online - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=online - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-07-44 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-07-44/.hydra/overrides.yaml b/outputs/2026-06-19/19-07-44/.hydra/overrides.yaml deleted file mode 100644 index 75166170fa8010ddd694dc7475a3046fac0c6f60..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-44/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- wandb.mode=online -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-07-44/main.log b/outputs/2026-06-19/19-07-44/main.log deleted file mode 100644 index f035b3c9e096a69b9d7cd63dc8316d413e1bec5a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-07-44/main.log +++ /dev/null @@ -1 +0,0 @@ -[2026-06-19 19:08:16,642][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 diff --git a/outputs/2026-06-19/19-13-42/.hydra/config.yaml b/outputs/2026-06-19/19-13-42/.hydra/config.yaml deleted file mode 100644 index dfda8864731f92d059d677bea2ca21a466ac793c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-42/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-42/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-42/.hydra/hydra.yaml deleted file mode 100644 index ced671ec869c943d624769e6482972cb1f27d06b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-42/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-42 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-42/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-42/.hydra/overrides.yaml deleted file mode 100644 index 0161cbdaa34e92ea4a75e133231c3c78fc6c89af..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-42/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-42/main.log b/outputs/2026-06-19/19-13-42/main.log deleted file mode 100644 index b7fa6138d5f569fab25ae79f2dcec75ff40dc3de..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-42/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 19:14:16,778][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-19 19:14:17,512][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-43/.hydra/config.yaml b/outputs/2026-06-19/19-13-43/.hydra/config.yaml deleted file mode 100644 index dfda8864731f92d059d677bea2ca21a466ac793c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-43/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-43/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-43/.hydra/hydra.yaml deleted file mode 100644 index a5742015f3712ac19e39d4e2250da75114636652..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-43/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-43 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-43/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-43/.hydra/overrides.yaml deleted file mode 100644 index 0161cbdaa34e92ea4a75e133231c3c78fc6c89af..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-43/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-43/main.log b/outputs/2026-06-19/19-13-43/main.log deleted file mode 100644 index 9af1185852f46a93e2441b83ae7b8c2260a1f52c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-43/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 19:14:16,858][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 19:14:16,906][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-44/.hydra/config.yaml b/outputs/2026-06-19/19-13-44/.hydra/config.yaml deleted file mode 100644 index dfda8864731f92d059d677bea2ca21a466ac793c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-44/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-44/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-44/.hydra/hydra.yaml deleted file mode 100644 index cc699184d7b57de5292515eb9b4e18d339513dda..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-44/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-44 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-44/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-44/.hydra/overrides.yaml deleted file mode 100644 index 0161cbdaa34e92ea4a75e133231c3c78fc6c89af..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-44/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-44/main.log b/outputs/2026-06-19/19-13-44/main.log deleted file mode 100644 index 241a0cf4709e30c29fa588095b9a58c925660a52..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-44/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-19 19:14:16,913][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-19 19:14:17,667][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:14:42,940][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-45/.hydra/config.yaml b/outputs/2026-06-19/19-13-45/.hydra/config.yaml deleted file mode 100644 index dfda8864731f92d059d677bea2ca21a466ac793c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-45/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-45/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-45/.hydra/hydra.yaml deleted file mode 100644 index 27ede79d9224b627ab11055c34164b1d7cf2dcb8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-45/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-45 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-45/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-45/.hydra/overrides.yaml deleted file mode 100644 index 0161cbdaa34e92ea4a75e133231c3c78fc6c89af..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-45/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-45/main.log b/outputs/2026-06-19/19-13-45/main.log deleted file mode 100644 index 9eed4d431638a5d715f7d04d0efd4a939721e1f5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-45/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 19:14:16,693][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 19:14:40,139][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:42,939][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-46/.hydra/config.yaml b/outputs/2026-06-19/19-13-46/.hydra/config.yaml deleted file mode 100644 index 5deab422580abeed5d76ece0f601c27f3c6f094d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-46/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-46/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-46/.hydra/hydra.yaml deleted file mode 100644 index 7400cf4ee0dc1966e12772b4b8377a2dd34f71fd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-46/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-46 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-46/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-46/.hydra/overrides.yaml deleted file mode 100644 index b3d602b2f3f21e413975c8c5b2dc7365caa1d6d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-46/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-46/main.log b/outputs/2026-06-19/19-13-46/main.log deleted file mode 100644 index b25ec9d42058db9b303d83751ef1e82103586170..0000000000000000000000000000000000000000 Binary files a/outputs/2026-06-19/19-13-46/main.log and /dev/null differ diff --git a/outputs/2026-06-19/19-13-47/.hydra/config.yaml b/outputs/2026-06-19/19-13-47/.hydra/config.yaml deleted file mode 100644 index 5deab422580abeed5d76ece0f601c27f3c6f094d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-47/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-47/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-47/.hydra/hydra.yaml deleted file mode 100644 index 101b48b9f2d33b73a4dfb36d3344e8d94faa0feb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-47/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-47 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-47/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-47/.hydra/overrides.yaml deleted file mode 100644 index b3d602b2f3f21e413975c8c5b2dc7365caa1d6d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-47/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-47/main.log b/outputs/2026-06-19/19-13-47/main.log deleted file mode 100644 index 2759a9f3f62cc623fff86e40edd3798af03f0076..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-47/main.log +++ /dev/null @@ -1,31 +0,0 @@ -[2026-06-19 19:14:19,872][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-19 19:14:20,017][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-19 19:14:20,018][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-19 19:14:20,018][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-19 19:14:20,018][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-19 19:14:20,018][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-19 19:14:20,139][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:14:20,262][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-19 19:14:21,926][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-19 19:14:22,502][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-19 19:14:28,347][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-19 19:14:46,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:46,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:47,865][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-19 19:14:49,766][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-19 19:14:49,766][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 04:51:09,030][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=175000` reached. diff --git a/outputs/2026-06-19/19-13-48/.hydra/config.yaml b/outputs/2026-06-19/19-13-48/.hydra/config.yaml deleted file mode 100644 index 5deab422580abeed5d76ece0f601c27f3c6f094d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-48/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-48/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-48/.hydra/hydra.yaml deleted file mode 100644 index e2b374663d6536b0eaa8ea3e4647c67387a2a917..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-48/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-48 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-48/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-48/.hydra/overrides.yaml deleted file mode 100644 index b3d602b2f3f21e413975c8c5b2dc7365caa1d6d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-48/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-48/main.log b/outputs/2026-06-19/19-13-48/main.log deleted file mode 100644 index 7e29bb92ca90dafe12df753e16355827c1825eb8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-48/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 19:14:21,858][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-19 19:14:46,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:49,766][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-49/.hydra/config.yaml b/outputs/2026-06-19/19-13-49/.hydra/config.yaml deleted file mode 100644 index 5deab422580abeed5d76ece0f601c27f3c6f094d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-49/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-49/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-49/.hydra/hydra.yaml deleted file mode 100644 index 1bf7253ffd6a5ff25c337048256e1462c9e0f861..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-49/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-49 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-49/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-49/.hydra/overrides.yaml deleted file mode 100644 index b3d602b2f3f21e413975c8c5b2dc7365caa1d6d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-49/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-49/main.log b/outputs/2026-06-19/19-13-49/main.log deleted file mode 100644 index 380ccad5f11aa594bdd828e0ce5f4d6878df5f73..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-49/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 19:14:21,872][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-19 19:14:46,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:49,766][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-19/19-13-50/.hydra/config.yaml b/outputs/2026-06-19/19-13-50/.hydra/config.yaml deleted file mode 100644 index 5deab422580abeed5d76ece0f601c27f3c6f094d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-50/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-19/19-13-50/.hydra/hydra.yaml b/outputs/2026-06-19/19-13-50/.hydra/hydra.yaml deleted file mode 100644 index 380cdf6ddad3a3324867b3dd44224af67e9a74c4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-50/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-19/19-13-50 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-19/19-13-50/.hydra/overrides.yaml b/outputs/2026-06-19/19-13-50/.hydra/overrides.yaml deleted file mode 100644 index b3d602b2f3f21e413975c8c5b2dc7365caa1d6d3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-50/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_base_a2d4r2_route_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-19/19-13-50/main.log b/outputs/2026-06-19/19-13-50/main.log deleted file mode 100644 index 9bbca6e29a2235602e1f69dd397ba6074dd9b622..0000000000000000000000000000000000000000 --- a/outputs/2026-06-19/19-13-50/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-19 19:14:22,041][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-19 19:14:46,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-19 19:14:49,766][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-19-07/.hydra/config.yaml b/outputs/2026-06-20/12-19-07/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-07/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-07/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-07/.hydra/hydra.yaml deleted file mode 100644 index d5222854d89c0beb0d69c2808081376b36cfb2cd..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-07/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-07 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-07/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-07/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-07/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-07/main.log b/outputs/2026-06-20/12-19-07/main.log deleted file mode 100644 index 5a266141d66cad05fb52ab8a42d5d370c4dbaf46..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-07/main.log +++ /dev/null @@ -1,10 +0,0 @@ -[2026-06-20 12:20:01,739][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-20 12:20:01,747][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:31,774][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 12:20:31,774][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,363][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-06-23 12:19:16,365][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-23 12:19:16,363][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-19-08/.hydra/config.yaml b/outputs/2026-06-20/12-19-08/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-08/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-08/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-08/.hydra/hydra.yaml deleted file mode 100644 index 3e95a942c12f1276cbd27800e58c67146dadd3d5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-08/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-08 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-08/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-08/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-08/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-08/main.log b/outputs/2026-06-20/12-19-08/main.log deleted file mode 100644 index fd16e739b82e89cfd25f470415fb0d1f2c302332..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-08/main.log +++ /dev/null @@ -1,34 +0,0 @@ -[2026-06-20 12:19:56,037][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-20 12:19:56,142][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-20 12:19:56,142][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-20 12:19:56,142][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-20 12:19:56,142][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-20 12:19:56,143][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-20 12:19:57,784][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 12:19:58,763][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 12:20:01,755][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-20 12:20:01,770][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-20 12:20:08,050][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:26,768][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-20 12:20:31,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 12:20:31,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-06-23 12:19:16,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-19-09/.hydra/config.yaml b/outputs/2026-06-20/12-19-09/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-09/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-09/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-09/.hydra/hydra.yaml deleted file mode 100644 index 66bab7c3a1cbc9adf2f272a0d9c9e3230613dbb2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-09/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-09 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-09/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-09/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-09/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-09/main.log b/outputs/2026-06-20/12-19-09/main.log deleted file mode 100644 index 8137993111a60f11808558c864bd586424d34b94..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-09/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-20 12:20:01,763][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:31,774][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-19-10/.hydra/config.yaml b/outputs/2026-06-20/12-19-10/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-10/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-10/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-10/.hydra/hydra.yaml deleted file mode 100644 index 70a9946891fd79adc4068dffc75b57c831cfe085..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-10/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-10 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-10/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-10/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-10/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-10/main.log b/outputs/2026-06-20/12-19-10/main.log deleted file mode 100644 index c76b764d24c5400ac81170aad783eb1c250e3761..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-10/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-20 12:20:01,758][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:31,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,363][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-06-23 12:19:16,365][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-19-11/.hydra/config.yaml b/outputs/2026-06-20/12-19-11/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-11/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-11/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-11/.hydra/hydra.yaml deleted file mode 100644 index 8c3fdf9da82d68c17a5108bc446ddebd912a6bb3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-11/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-11 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-11/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-11/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-11/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-11/main.log b/outputs/2026-06-20/12-19-11/main.log deleted file mode 100644 index b446545b4010640893e3a257619bbddf6ce64e1f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-11/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-20 12:20:01,753][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:31,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,365][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-19-12/.hydra/config.yaml b/outputs/2026-06-20/12-19-12/.hydra/config.yaml deleted file mode 100644 index e20b9506faae86321b20e5ef9bdd6bf0c914b072..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-12/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 8 - max_epochs: -1 - max_steps: 350000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: false - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 4.0e-05 - memory_adapters: 4.0e-05 - full_dit: 1.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-19-12/.hydra/hydra.yaml b/outputs/2026-06-20/12-19-12/.hydra/hydra.yaml deleted file mode 100644 index a1ac19d5257f184e212e87371a79c7b5c9e170c1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-12/.hydra/hydra.yaml +++ /dev/null @@ -1,218 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - algorithm.diffusion.use_fused_snr=false - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 - - experiment.training.batch_size=8 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=350000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.diffusion.use_fused_snr=false,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=8,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=350000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-19-12 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-19-12/.hydra/overrides.yaml b/outputs/2026-06-20/12-19-12/.hydra/overrides.yaml deleted file mode 100644 index 270d3b697f2e4bc4f82664afd6d5ebcd6d161d0a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-12/.hydra/overrides.yaml +++ /dev/null @@ -1,60 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_snr_shift_a2d4r2_route_ntok8_8a100_bs8_global64_350k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- algorithm.diffusion.use_fused_snr=false -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=4.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=1.0e-5 -- experiment.training.batch_size=8 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=350000 diff --git a/outputs/2026-06-20/12-19-12/main.log b/outputs/2026-06-20/12-19-12/main.log deleted file mode 100644 index 1f9781425798c787e961aeb5ca5f22b887ec4f71..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-19-12/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-20 12:20:01,739][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-20 12:20:24,697][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 12:20:31,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-23 12:19:16,365][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-06-23 12:19:16,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-20/12-58-49/.hydra/config.yaml b/outputs/2026-06-20/12-58-49/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-49/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-49/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-49/.hydra/hydra.yaml deleted file mode 100644 index 0e9c4e5d469397dcff761e49c379737a65467042..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-49/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-49 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-49/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-49/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-49/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-49/main.log b/outputs/2026-06-20/12-58-49/main.log deleted file mode 100644 index b1c1faceab1b39385f5ca9436866551d3867d02a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-49/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 12:59:40,037][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-58-50/.hydra/config.yaml b/outputs/2026-06-20/12-58-50/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-50/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-50/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-50/.hydra/hydra.yaml deleted file mode 100644 index a2ae8bdeedbae8d1654cfad41ab4a28dea502844..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-50/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-50 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-50/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-50/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-50/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-50/main.log b/outputs/2026-06-20/12-58-50/main.log deleted file mode 100644 index 3989993889545f9f177f2be8e8d8041e04a667db..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-50/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-20 12:59:40,037][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-20 12:59:40,065][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-58-51/.hydra/config.yaml b/outputs/2026-06-20/12-58-51/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-51/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-51/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-51/.hydra/hydra.yaml deleted file mode 100644 index d24648af57a038a1fa9b9df9064704dbe764cbd0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-51/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-51 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-51/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-51/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-51/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-51/main.log b/outputs/2026-06-20/12-58-51/main.log deleted file mode 100644 index d6127192219347fbb38e29ede81d09ea1eaa8291..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-51/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 12:59:40,053][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-58-52/.hydra/config.yaml b/outputs/2026-06-20/12-58-52/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-52/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-52/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-52/.hydra/hydra.yaml deleted file mode 100644 index 3acc6009416619afaaaa148576d1b946075d818b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-52/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-52 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-52/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-52/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-52/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-52/main.log b/outputs/2026-06-20/12-58-52/main.log deleted file mode 100644 index 8e85f54dfb3ad9bf0ca00c735c807c08a4ebe359..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-52/main.log +++ /dev/null @@ -1,28 +0,0 @@ -[2026-06-20 12:59:33,702][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-20 12:59:34,857][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-20 12:59:34,857][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-20 12:59:34,857][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-20 12:59:34,857][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-20 12:59:34,858][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-20 12:59:36,657][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 12:59:37,984][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 12:59:40,037][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-20 12:59:45,715][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:05,892][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 11:10:13,140][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=175000` reached. diff --git a/outputs/2026-06-20/12-58-53/.hydra/config.yaml b/outputs/2026-06-20/12-58-53/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-53/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-53/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-53/.hydra/hydra.yaml deleted file mode 100644 index e1430318fe06992f5a6e80b17d96553e8f1fabd6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-53/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-53 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-53/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-53/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-53/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-53/main.log b/outputs/2026-06-20/12-58-53/main.log deleted file mode 100644 index a2fe57bed5bf361d592796785a49776f21e9595b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-53/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 12:59:40,051][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-58-54/.hydra/config.yaml b/outputs/2026-06-20/12-58-54/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-54/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-54/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-54/.hydra/hydra.yaml deleted file mode 100644 index 0b6e6a42835784bbb44e93f527394a1ad6b4b98f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-54/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-54 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-54/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-54/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-54/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-54/main.log b/outputs/2026-06-20/12-58-54/main.log deleted file mode 100644 index bc38b646c4fcb1a1f4aac0b573033d145c523359..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-54/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 12:59:40,037][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/12-58-55/.hydra/config.yaml b/outputs/2026-06-20/12-58-55/.hydra/config.yaml deleted file mode 100644 index faf4fdc2cdedc288bdd5309e6df8362385b2f4ea..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-55/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: all - dynamic: all - revisit: all - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/12-58-55/.hydra/hydra.yaml b/outputs/2026-06-20/12-58-55/.hydra/hydra.yaml deleted file mode 100644 index e37a4fc2120f4967d212e9a378998c194b0ed511..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-55/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=all - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=all - - ++algorithm.dememwm.frame_memory.noise_route.revisit=all - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=0,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=all,++algorithm.dememwm.frame_memory.noise_route.dynamic=all,++algorithm.dememwm.frame_memory.noise_route.revisit=all,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/12-58-55 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/12-58-55/.hydra/overrides.yaml b/outputs/2026-06-20/12-58-55/.hydra/overrides.yaml deleted file mode 100644 index 9ea76abe63e906884ccf35decf29b5499e51412d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-55/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_no_noise_route_no_dynamic_a2r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=0 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=all -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=all -- ++algorithm.dememwm.frame_memory.noise_route.revisit=all -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/12-58-55/main.log b/outputs/2026-06-20/12-58-55/main.log deleted file mode 100644 index 312bc16f3e67b045f4c3c2ab47992e8cde200eaf..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/12-58-55/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 12:59:40,037][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-20 13:00:04,596][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:00:07,035][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/13-04-59/.hydra/config.yaml b/outputs/2026-06-20/13-04-59/.hydra/config.yaml deleted file mode 100644 index 4981e7b728f86c287b16ee6a10121000a82e701e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-04-59/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: low - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/13-04-59/.hydra/hydra.yaml b/outputs/2026-06-20/13-04-59/.hydra/hydra.yaml deleted file mode 100644 index 48c711d5fe20dbdd2d55c3e8393b4631f8a77644..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-04-59/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=low - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=low,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/13-04-59 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/13-04-59/.hydra/overrides.yaml b/outputs/2026-06-20/13-04-59/.hydra/overrides.yaml deleted file mode 100644 index 07967ff3a9545c962a89eaa7556655ddd496dad9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-04-59/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=low -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/13-04-59/main.log b/outputs/2026-06-20/13-04-59/main.log deleted file mode 100644 index 344b4ab2839779fd8b92c3935cf0247a2a9921c5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-04-59/main.log +++ /dev/null @@ -1,40 +0,0 @@ -[2026-06-20 13:05:47,204][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-20 13:05:47,479][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-20 13:05:47,479][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-20 13:05:47,479][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-20 13:05:47,479][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-20 13:05:47,479][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-20 13:05:49,903][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: matched=302 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 13:05:51,092][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: matched=228 missing=0 shape_mismatch=0 prefix='' -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-20 13:05:53,484][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-20 13:05:53,500][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-20 13:05:58,974][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:19,956][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -1.0 B Trainable params -231 M Non-trainable params -1.3 B Total params -5,107.144 Total estimated model params size (MB) -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-22 21:41:21,456][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=175000` reached. diff --git a/outputs/2026-06-20/13-05-00/.hydra/config.yaml b/outputs/2026-06-20/13-05-00/.hydra/config.yaml deleted file mode 100644 index 4981e7b728f86c287b16ee6a10121000a82e701e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-00/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: low - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/13-05-00/.hydra/hydra.yaml b/outputs/2026-06-20/13-05-00/.hydra/hydra.yaml deleted file mode 100644 index c4f5f2088d9b629296440f3c226f00a1b8dbc897..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-00/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=low - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=low,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/13-05-00 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/13-05-00/.hydra/overrides.yaml b/outputs/2026-06-20/13-05-00/.hydra/overrides.yaml deleted file mode 100644 index 07967ff3a9545c962a89eaa7556655ddd496dad9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-00/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=low -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/13-05-00/main.log b/outputs/2026-06-20/13-05-00/main.log deleted file mode 100644 index 5d9fe1fe0f2bd836b1c7b88a1cdfc0c4a95945ba..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-00/main.log +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-20/13-05-01/.hydra/config.yaml b/outputs/2026-06-20/13-05-01/.hydra/config.yaml deleted file mode 100644 index 4981e7b728f86c287b16ee6a10121000a82e701e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-01/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 175000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - save_last: true - enable_version_counter: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: true - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - metadata: data/${dataset.name}/metadata.json - data_mean: 0.5 - data_std: 0.5 - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - n_frames: 1000 - context_length: 1 - resolution: 128 - observation_shape: - - 3 - - ${dataset.resolution} - - ${dataset.resolution} - external_cond_dim: 0 - validation_multiplier: 1 - frame_skip: 1 - action_cond_dim: 25 - memory_condition_length: 0 - customized_validation: true - angle_range: 180 - pos_range: 8 - max_getitem_retries: 32 - _name: video_minecraft_latent - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - n_frames_valid: 500 - wo_updown: false -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: - - 16 - - 18 - - 32 - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 100 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - action_cond_dim: 25 - use_plucker: true - _name: dememwm_frame_memory_ablation - memory_token_cross_attention: false - pose_cond_dim: 5 - log_video: true - dememwm: - enabled: true - debug_stdout_metrics: true - training_stage: stage_2 - curriculum: - enabled: true - full_stage_start_step: 0 - freeze_vae: true - lr: - dememwm_modules: 8.0e-05 - memory_adapters: 8.0e-05 - full_dit: 2.0e-05 - frame_memory: - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - dynamic_temporal: bidirectional_stream - anchor_temporal: diagonal - revisit_temporal: diagonal - noise_route: - anchor: high - dynamic: low - revisit: low - anchor_diverse_selection: true - chunk_retrieval: - chunk_fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - allow_empty_revisit: true - candidate_chunk_size: 64 - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - revisit_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit: - deterministic_pose_retrieval: true - fov_half_h: 52.5 - fov_half_v: 37.5 - fov_radius: 30.0 - fov_yaw_samples: 25 - fov_pitch_samples: 20 - fov_depth_samples: 20 - pose_preselect_topk: 64 - plucker_weight: 0.1 - plucker_grid_h: 4 - plucker_grid_w: 4 - plucker_focal_length: 0.35 - revisit_metric: - enabled: false - min_gap: 8 - gap_bands: - - 32 - - 128 - fov_overlap_threshold: 0.6 - positional_distance_threshold: 2.0 - self_crop_fraction: 0.5 - cache: - enabled: false - checkpoint: - strict_dememwm_eval_load: true - n_tokens: 8 -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -only_tune_memory: false diff --git a/outputs/2026-06-20/13-05-01/.hydra/hydra.yaml b/outputs/2026-06-20/13-05-01/.hydra/hydra.yaml deleted file mode 100644 index dd7bf506276880e476bb7790deb98c60f70876d5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-01/.hydra/hydra.yaml +++ /dev/null @@ -1,217 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ - - auto_resume=true - - experiment.tasks=[training] - - algorithm=dememwm_frame_memory_ablation - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - +only_tune_memory=false - - dataset=video_minecraft_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=1000 - - +dataset.n_frames_valid=500 - - dataset.customized_validation=true - - +dataset.wo_updown=false - - dataset.angle_range=180 - - dataset.pos_range=8 - - ++algorithm.n_tokens=8 - - algorithm.x_shape=[16,18,32] - - ++algorithm.context_frames=100 - - ++algorithm.log_video=true - - ++algorithm.diffusion.sampling_timesteps=20 - - ++algorithm.dememwm.debug_stdout_metrics=true - - ++algorithm.dememwm.revisit_metric.enabled=false - - ++algorithm.dememwm.frame_memory.max_anchor_frames=2 - - ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 - - ++algorithm.dememwm.frame_memory.max_revisit_frames=2 - - ++algorithm.dememwm.frame_memory.noise_route.anchor=high - - ++algorithm.dememwm.frame_memory.noise_route.dynamic=low - - ++algorithm.dememwm.frame_memory.noise_route.revisit=low - - ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true - - ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 - - ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 - - ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true - - ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 - - ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 - - ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 - - ++algorithm.dememwm.revisit.fov_yaw_samples=25 - - ++algorithm.dememwm.revisit.fov_pitch_samples=20 - - ++algorithm.dememwm.revisit.fov_depth_samples=20 - - ++algorithm.dememwm.revisit.plucker_weight=0.10 - - ++algorithm.dememwm.revisit.plucker_focal_length=0.35 - - ++algorithm.dememwm.revisit.pose_preselect_topk=64 - - ++algorithm.dememwm.curriculum.enabled=true - - ++algorithm.dememwm.curriculum.full_stage_start_step=0 - - ++algorithm.dememwm.curriculum.freeze_vae=true - - ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 - - ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=175000 - job: - name: main - chdir: null - override_dirname: ++algorithm.context_frames=100,++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true,++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64,++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60,++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10,++algorithm.dememwm.curriculum.enabled=true,++algorithm.dememwm.curriculum.freeze_vae=true,++algorithm.dememwm.curriculum.full_stage_start_step=0,++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5,++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5,++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5,++algorithm.dememwm.debug_stdout_metrics=true,++algorithm.dememwm.frame_memory.anchor_diverse_selection=true,++algorithm.dememwm.frame_memory.max_anchor_frames=2,++algorithm.dememwm.frame_memory.max_dynamic_frames=4,++algorithm.dememwm.frame_memory.max_revisit_frames=2,++algorithm.dememwm.frame_memory.noise_route.anchor=high,++algorithm.dememwm.frame_memory.noise_route.dynamic=low,++algorithm.dememwm.frame_memory.noise_route.revisit=low,++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25,++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50,++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25,++algorithm.dememwm.revisit.fov_depth_samples=20,++algorithm.dememwm.revisit.fov_pitch_samples=20,++algorithm.dememwm.revisit.fov_yaw_samples=25,++algorithm.dememwm.revisit.plucker_focal_length=0.35,++algorithm.dememwm.revisit.plucker_weight=0.10,++algorithm.dememwm.revisit.pose_preselect_topk=64,++algorithm.dememwm.revisit_metric.enabled=false,++algorithm.diffusion.sampling_timesteps=20,++algorithm.log_video=true,++algorithm.n_tokens=8,+customized_load=true,+dataset.n_frames_valid=500,+dataset.wo_updown=false,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k,+only_tune_memory=false,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.x_shape=[16,18,32],algorithm=dememwm_frame_memory_ablation,auto_resume=true,dataset.angle_range=180,dataset.customized_validation=true,dataset.n_frames=1000,dataset.pos_range=8,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset=video_minecraft_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.max_steps=175000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000 - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-20/13-05-01 - choices: - cluster: null - algorithm: dememwm_frame_memory_ablation - dataset: video_minecraft_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-20/13-05-01/.hydra/overrides.yaml b/outputs/2026-06-20/13-05-01/.hydra/overrides.yaml deleted file mode 100644 index 07967ff3a9545c962a89eaa7556655ddd496dad9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-01/.hydra/overrides.yaml +++ /dev/null @@ -1,59 +0,0 @@ -- +name=train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sep_mem_fulldit_mem_route_anchor_high_dynamic_revisit_low_a2d4r2_ntok8_8h200_bs16_global128_175k/ -- auto_resume=true -- experiment.tasks=[training] -- algorithm=dememwm_frame_memory_ablation -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- +only_tune_memory=false -- dataset=video_minecraft_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=1000 -- +dataset.n_frames_valid=500 -- dataset.customized_validation=true -- +dataset.wo_updown=false -- dataset.angle_range=180 -- dataset.pos_range=8 -- ++algorithm.n_tokens=8 -- algorithm.x_shape=[16,18,32] -- ++algorithm.context_frames=100 -- ++algorithm.log_video=true -- ++algorithm.diffusion.sampling_timesteps=20 -- ++algorithm.dememwm.debug_stdout_metrics=true -- ++algorithm.dememwm.revisit_metric.enabled=false -- ++algorithm.dememwm.frame_memory.max_anchor_frames=2 -- ++algorithm.dememwm.frame_memory.max_dynamic_frames=4 -- ++algorithm.dememwm.frame_memory.max_revisit_frames=2 -- ++algorithm.dememwm.frame_memory.noise_route.anchor=high -- ++algorithm.dememwm.frame_memory.noise_route.dynamic=low -- ++algorithm.dememwm.frame_memory.noise_route.revisit=low -- ++algorithm.dememwm.frame_memory.anchor_diverse_selection=true -- ++algorithm.dememwm.chunk_retrieval.chunk_fov_overlap_threshold=0.60 -- ++algorithm.dememwm.chunk_retrieval.min_total_selected_coverage=0.10 -- ++algorithm.dememwm.chunk_retrieval.allow_empty_revisit=true -- ++algorithm.dememwm.chunk_retrieval.candidate_chunk_size=64 -- ++algorithm.dememwm.memory_noise.anchor_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.revisit_max_fraction=0.25 -- ++algorithm.dememwm.memory_noise.dynamic_max_fraction=0.50 -- ++algorithm.dememwm.revisit.fov_yaw_samples=25 -- ++algorithm.dememwm.revisit.fov_pitch_samples=20 -- ++algorithm.dememwm.revisit.fov_depth_samples=20 -- ++algorithm.dememwm.revisit.plucker_weight=0.10 -- ++algorithm.dememwm.revisit.plucker_focal_length=0.35 -- ++algorithm.dememwm.revisit.pose_preselect_topk=64 -- ++algorithm.dememwm.curriculum.enabled=true -- ++algorithm.dememwm.curriculum.full_stage_start_step=0 -- ++algorithm.dememwm.curriculum.freeze_vae=true -- ++algorithm.dememwm.curriculum.lr.dememwm_modules=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.memory_adapters=8.0e-5 -- ++algorithm.dememwm.curriculum.lr.full_dit=2.0e-5 -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=175000 diff --git a/outputs/2026-06-20/13-05-01/main.log b/outputs/2026-06-20/13-05-01/main.log deleted file mode 100644 index a347d9cb474fc066c6039ad6a9bdec7762dfa054..0000000000000000000000000000000000000000 --- a/outputs/2026-06-20/13-05-01/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-06-20 13:05:53,479][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-20 13:06:18,966][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-20 13:06:22,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-25/20-41-58/.hydra/config.yaml b/outputs/2026-06-25/20-41-58/.hydra/config.yaml deleted file mode 100644 index 7c97550b32338ca18b91468ee081745e758dd303..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-58/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: recent - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-41-58/.hydra/hydra.yaml b/outputs/2026-06-25/20-41-58/.hydra/hydra.yaml deleted file mode 100644 index 68b652c16c00981e19a3056a0b6248bb532f7cc4..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-58/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=recent - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=recent,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-41-58 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-41-58/.hydra/overrides.yaml b/outputs/2026-06-25/20-41-58/.hydra/overrides.yaml deleted file mode 100644 index 634c9c262ff8c9a6fac58a418e60d4661df1695a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-58/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=recent -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-41-58/main.log b/outputs/2026-06-25/20-41-58/main.log deleted file mode 100644 index 8a464c9bda4df31b449317d213b4baad503a4d7c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-58/main.log +++ /dev/null @@ -1,87 +0,0 @@ -[2026-06-25 20:42:51,361][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:42:51,362][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-25 20:42:53,680][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-25 20:42:53,680][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-25 20:42:53,923][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:42:54,998][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-25 20:42:56,277][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-25 20:42:56,419][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-25 20:42:56,419][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-25 20:42:56,419][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-25 20:42:56,419][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-25 20:42:56,419][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-25 20:43:50,238][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-25 20:43:50,239][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-25 20:43:50,758][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-25 20:43:57,072][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:18,116][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-25 20:44:20,911][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 22:04:28,124][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:04:28,866][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:04:28,867][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:04:28,867][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 13:48:08,874][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 13:48:40,202][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,984][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 13:48:40,984][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 13:48:40,984][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 13:48:40,984][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 13:48:40,985][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 13:50:40,309][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints exists and is not empty. - -[2026-06-26 13:50:40,310][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:50:40,329][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,331][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,334][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:51:01,538][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,648][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,656][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:02,797][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 13:51:03,960][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,135][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:06,389][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-06-25/20-41-59/.hydra/config.yaml b/outputs/2026-06-25/20-41-59/.hydra/config.yaml deleted file mode 100644 index 7c97550b32338ca18b91468ee081745e758dd303..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-59/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: recent - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-41-59/.hydra/hydra.yaml b/outputs/2026-06-25/20-41-59/.hydra/hydra.yaml deleted file mode 100644 index 4437b5cdc8ee17d2a3d0579efc896041bc309b2b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-59/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=recent - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=recent,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-41-59 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-41-59/.hydra/overrides.yaml b/outputs/2026-06-25/20-41-59/.hydra/overrides.yaml deleted file mode 100644 index 634c9c262ff8c9a6fac58a418e60d4661df1695a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-59/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=recent -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-41-59/main.log b/outputs/2026-06-25/20-41-59/main.log deleted file mode 100644 index c35b0fda4f4b1abe53640231fed351fc88414209..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-41-59/main.log +++ /dev/null @@ -1,45 +0,0 @@ -[2026-06-25 20:43:50,238][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-25 20:43:50,241][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-25 20:43:50,251][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-25 20:43:50,260][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-25 20:43:51,094][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:16,384][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:44:20,911][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:44:20,912][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,202][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:48:40,203][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:50:40,317][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,323][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,325][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,327][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:50:40,336][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:51:01,558][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,573][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,662][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,664][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:01,750][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,134][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:51:04,135][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-25/20-46-55/.hydra/config.yaml b/outputs/2026-06-25/20-46-55/.hydra/config.yaml deleted file mode 100644 index 7021a9adaba2f265eb735cd2e1ba34e1b0e98274..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-55/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-46-55/.hydra/hydra.yaml b/outputs/2026-06-25/20-46-55/.hydra/hydra.yaml deleted file mode 100644 index 3257555fd7dec6955945b2ca5d1a36b84cbd01e2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-55/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-46-55 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-46-55/.hydra/overrides.yaml b/outputs/2026-06-25/20-46-55/.hydra/overrides.yaml deleted file mode 100644 index 50839acd41803ee9a5606af6358b422484826767..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-55/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-46-55/main.log b/outputs/2026-06-25/20-46-55/main.log deleted file mode 100644 index 48cc19ef311ff80b70541d2c34db07b18b07ad6c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-55/main.log +++ /dev/null @@ -1,18 +0,0 @@ -[2026-06-25 20:48:35,989][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-25 20:48:36,069][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:55:02,317][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,319][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:56:50,190][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,193][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:10,231][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,343][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-25/20-46-56/.hydra/config.yaml b/outputs/2026-06-25/20-46-56/.hydra/config.yaml deleted file mode 100644 index 7021a9adaba2f265eb735cd2e1ba34e1b0e98274..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-56/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-46-56/.hydra/hydra.yaml b/outputs/2026-06-25/20-46-56/.hydra/hydra.yaml deleted file mode 100644 index 947816c6fc5eca2c3fcd6955be6425ada8871ca2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-56/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-46-56 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-46-56/.hydra/overrides.yaml b/outputs/2026-06-25/20-46-56/.hydra/overrides.yaml deleted file mode 100644 index 50839acd41803ee9a5606af6358b422484826767..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-56/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-46-56/main.log b/outputs/2026-06-25/20-46-56/main.log deleted file mode 100644 index b3091bb38a0087a912cc65d3af471e9ed1baedb3..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-46-56/main.log +++ /dev/null @@ -1,114 +0,0 @@ -[2026-06-25 20:47:42,438][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:47:42,439][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-25 20:47:44,318][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-25 20:47:44,319][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-25 20:47:44,523][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:47:45,539][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-25 20:47:47,472][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-25 20:47:47,621][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-25 20:47:47,621][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-25 20:47:47,621][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-25 20:47:47,621][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-25 20:47:47,621][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-25 20:48:35,983][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-25 20:48:35,983][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-25 20:48:35,984][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-25 20:48:35,984][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-25 20:48:35,985][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-25 20:48:35,987][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-25 20:48:41,626][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:00,760][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:49:03,680][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:49:04,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 22:09:23,939][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:09:25,688][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:09:25,688][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:09:25,688][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 13:54:35,992][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 13:55:02,317][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 13:55:02,317][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,317][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,317][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,318][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,319][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:02,319][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:03,826][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 13:55:03,826][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 13:55:03,826][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 13:55:03,826][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 13:55:03,827][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 13:56:50,180][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints exists and is not empty. - -[2026-06-26 13:56:50,181][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:56:50,188][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,195][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,199][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,201][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:56:50,202][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:10,226][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,232][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,234][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,237][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,238][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:10,241][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:11,151][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 13:57:12,280][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:12,313][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:13,295][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-06-25/20-48-27/.hydra/config.yaml b/outputs/2026-06-25/20-48-27/.hydra/config.yaml deleted file mode 100644 index 3e0d1eacb4a756bcfda7d47203d6772fdda4254a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-27/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: false - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-48-27/.hydra/hydra.yaml b/outputs/2026-06-25/20-48-27/.hydra/hydra.yaml deleted file mode 100644 index 04f338bb45c36606bcf0ad6ac57403461f5cc2eb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-27/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-48-27 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-48-27/.hydra/overrides.yaml b/outputs/2026-06-25/20-48-27/.hydra/overrides.yaml deleted file mode 100644 index 5c8056a58520299ba8204bad4bbcc640e272ce00..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-27/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-48-27/main.log b/outputs/2026-06-25/20-48-27/main.log deleted file mode 100644 index e3243c1a33b535af1489abd14b9e3877714b2c8f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-27/main.log +++ /dev/null @@ -1,123 +0,0 @@ -[2026-06-25 20:49:16,230][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:49:16,231][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-25 20:49:18,375][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-25 20:49:18,375][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-25 20:49:18,540][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 20:49:19,633][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-25 20:49:20,614][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-25 20:49:20,769][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-25 20:49:20,769][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-25 20:49:20,769][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-25 20:49:20,769][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-25 20:49:20,770][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-25 20:50:10,880][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-25 20:50:10,880][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-25 20:50:10,881][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-25 20:50:10,897][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-25 20:50:10,913][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-25 20:50:10,922][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-25 20:50:11,941][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-25 20:50:18,975][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:39,913][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 22:10:55,981][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:11:02,035][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:11:02,035][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:11:02,036][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 13:54:55,308][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 13:55:15,902][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:55:16,851][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 13:55:16,851][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 13:55:16,851][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 13:55:16,851][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 13:55:16,852][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 13:57:08,336][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints exists and is not empty. - -[2026-06-26 13:57:08,337][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:57:08,347][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,349][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,351][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,353][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,355][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,356][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:08,358][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:27,190][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,192][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,195][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,198][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,199][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,204][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:27,307][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:28,097][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 13:57:29,255][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:57:30,277][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-06-25/20-48-28/.hydra/config.yaml b/outputs/2026-06-25/20-48-28/.hydra/config.yaml deleted file mode 100644 index 3e0d1eacb4a756bcfda7d47203d6772fdda4254a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-28/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: false - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/20-48-28/.hydra/hydra.yaml b/outputs/2026-06-25/20-48-28/.hydra/hydra.yaml deleted file mode 100644 index 5f79c7d1c86b842405a68921f03971491b3bc969..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-28/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/20-48-28 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/20-48-28/.hydra/overrides.yaml b/outputs/2026-06-25/20-48-28/.hydra/overrides.yaml deleted file mode 100644 index 5c8056a58520299ba8204bad4bbcc640e272ce00..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-28/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/20-48-28/main.log b/outputs/2026-06-25/20-48-28/main.log deleted file mode 100644 index d0f354cdeaa2caeb90386f108b9a8528bd1e7da2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/20-48-28/main.log +++ /dev/null @@ -1,9 +0,0 @@ -[2026-06-25 20:50:10,882][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-25 20:50:37,831][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 20:50:39,915][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 13:55:15,902][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 13:57:08,342][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 13:57:27,198][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 13:57:29,263][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-25/21-28-59/.hydra/config.yaml b/outputs/2026-06-25/21-28-59/.hydra/config.yaml deleted file mode 100644 index d24738b99d91e3ae14cbbe8a4c211bcf4b2fc76b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-28-59/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/21-28-59/.hydra/hydra.yaml b/outputs/2026-06-25/21-28-59/.hydra/hydra.yaml deleted file mode 100644 index ed19374bfb472c12c6ce2d904c84bd2fe664bd0e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-28-59/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/21-28-59 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/21-28-59/.hydra/overrides.yaml b/outputs/2026-06-25/21-28-59/.hydra/overrides.yaml deleted file mode 100644 index 4b85025abd8fab687599f0ac1ed0ad4c8ddb9f9a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-28-59/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/21-28-59/main.log b/outputs/2026-06-25/21-28-59/main.log deleted file mode 100644 index 924f3eab48ffbf6024f644a2a64e88d678d0e23b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-28-59/main.log +++ /dev/null @@ -1,123 +0,0 @@ -[2026-06-25 21:29:44,339][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 21:29:44,339][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-25 21:29:44,904][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-25 21:29:44,905][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-25 21:29:45,157][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-25 21:29:46,389][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-25 21:29:47,572][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-25 21:29:47,704][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-25 21:29:47,704][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-25 21:29:47,704][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-25 21:29:47,704][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-25 21:29:47,705][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-25 21:30:32,154][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-25 21:30:32,154][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-25 21:30:32,154][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-25 21:30:32,154][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-25 21:30:32,155][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-25 21:30:32,174][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-25 21:30:32,439][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-25 21:30:38,009][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:57,269][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 21:30:58,776][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-25 22:51:12,834][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:51:17,265][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:51:17,265][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-25 22:51:17,265][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 14:36:39,200][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 14:37:00,532][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,533][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 14:37:00,533][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,533][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,533][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,533][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,534][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,534][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:37:00,869][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 14:37:00,869][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 14:37:00,869][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 14:37:00,869][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 14:37:00,870][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 14:38:39,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints exists and is not empty. - -[2026-06-26 14:38:39,386][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 14:38:39,390][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,395][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,397][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,398][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,400][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,402][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:38:39,404][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:39:01,288][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,292][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,293][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,293][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,310][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,310][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:01,317][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:02,200][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 14:39:03,357][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:39:05,104][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-06-25/21-29-00/.hydra/config.yaml b/outputs/2026-06-25/21-29-00/.hydra/config.yaml deleted file mode 100644 index d24738b99d91e3ae14cbbe8a4c211bcf4b2fc76b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-29-00/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-25/21-29-00/.hydra/hydra.yaml b/outputs/2026-06-25/21-29-00/.hydra/hydra.yaml deleted file mode 100644 index 4067908c5fe8b79bcd68caab28d333abcbaebb08..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-29-00/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-25/21-29-00 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-25/21-29-00/.hydra/overrides.yaml b/outputs/2026-06-25/21-29-00/.hydra/overrides.yaml deleted file mode 100644 index 4b85025abd8fab687599f0ac1ed0ad4c8ddb9f9a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-29-00/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-25/21-29-00/main.log b/outputs/2026-06-25/21-29-00/main.log deleted file mode 100644 index f7520d551c52b52c7a0d750a73fdb606ba34ea3c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-25/21-29-00/main.log +++ /dev/null @@ -1,9 +0,0 @@ -[2026-06-25 21:30:32,157][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-25 21:30:56,337][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-25 21:30:58,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 14:37:00,534][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-06-26 14:38:39,396][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-06-26 14:39:01,295][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 14:39:03,399][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-26/21-54-56/.hydra/config.yaml b/outputs/2026-06-26/21-54-56/.hydra/config.yaml deleted file mode 100644 index 7c97550b32338ca18b91468ee081745e758dd303..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-56/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: recent - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-26/21-54-56/.hydra/hydra.yaml b/outputs/2026-06-26/21-54-56/.hydra/hydra.yaml deleted file mode 100644 index a6b58f77abfd2e100b4c099608e303f2f5e58d4e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-56/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=recent - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=recent,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-26/21-54-56 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-26/21-54-56/.hydra/overrides.yaml b/outputs/2026-06-26/21-54-56/.hydra/overrides.yaml deleted file mode 100644 index 634c9c262ff8c9a6fac58a418e60d4661df1695a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-56/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=recent -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-26/21-54-56/main.log b/outputs/2026-06-26/21-54-56/main.log deleted file mode 100644 index 4ba56b169df0b8bc2bf8919af62a33d1b08c928c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-56/main.log +++ /dev/null @@ -1,25 +0,0 @@ -[2026-06-26 21:55:58,068][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-26 21:55:58,070][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-26 21:55:58,084][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-26 21:55:58,098][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-26 21:55:58,106][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-26 21:56:44,104][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,112][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,124][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,136][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,226][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:52,480][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,482][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,484][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,489][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,533][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-26/21-54-57/.hydra/config.yaml b/outputs/2026-06-26/21-54-57/.hydra/config.yaml deleted file mode 100644 index 7c97550b32338ca18b91468ee081745e758dd303..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-57/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: recent - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-26/21-54-57/.hydra/hydra.yaml b/outputs/2026-06-26/21-54-57/.hydra/hydra.yaml deleted file mode 100644 index 43d9cc116978e091425c5889505fed36a915306f..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-57/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=recent - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=recent,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-26/21-54-57 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-26/21-54-57/.hydra/overrides.yaml b/outputs/2026-06-26/21-54-57/.hydra/overrides.yaml deleted file mode 100644 index 634c9c262ff8c9a6fac58a418e60d4661df1695a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-57/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=recent -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-26/21-54-57/main.log b/outputs/2026-06-26/21-54-57/main.log deleted file mode 100644 index ac44034ee605512e0efcf6519b1f71ae51b043f2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-54-57/main.log +++ /dev/null @@ -1,68 +0,0 @@ -[2026-06-26 21:55:29,654][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 21:55:29,778][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 21:55:29,778][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 21:55:29,778][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 21:55:29,778][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 21:55:29,779][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 21:55:58,064][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-26 21:55:58,079][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-26 21:55:58,127][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-26 21:56:05,222][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-26 21:56:20,976][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:56:44,107][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,127][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:44,131][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:56:47,599][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 21:56:48,651][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:49,709][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:56:51,227][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 21:57:14,412][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 21:57:15,913][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 21:57:15,914][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 21:57:15,914][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 21:57:15,914][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 21:57:15,914][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 21:58:29,958][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:58:52,478][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,479][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:52,488][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:58:53,295][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 21:58:54,456][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_base_recent_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:58:54,484][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:01:33,780][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:01:33,793][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:01:33,794][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:01:33,794][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-28 15:39:11,810][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=200000` reached. diff --git a/outputs/2026-06-26/21-55-27/.hydra/config.yaml b/outputs/2026-06-26/21-55-27/.hydra/config.yaml deleted file mode 100644 index 7021a9adaba2f265eb735cd2e1ba34e1b0e98274..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-55-27/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-26/21-55-27/.hydra/hydra.yaml b/outputs/2026-06-26/21-55-27/.hydra/hydra.yaml deleted file mode 100644 index 4a7e587cecad34f3002453281110566ea0b7b149..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-55-27/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-26/21-55-27 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-26/21-55-27/.hydra/overrides.yaml b/outputs/2026-06-26/21-55-27/.hydra/overrides.yaml deleted file mode 100644 index 50839acd41803ee9a5606af6358b422484826767..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-55-27/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-26/21-55-27/main.log b/outputs/2026-06-26/21-55-27/main.log deleted file mode 100644 index 21c0c418b63718bca8debeb4909d6a9a5a5c909b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/21-55-27/main.log +++ /dev/null @@ -1,93 +0,0 @@ -[2026-06-26 21:56:15,375][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 21:56:15,513][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 21:56:15,514][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 21:56:15,514][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 21:56:15,514][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 21:56:15,514][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 21:57:03,263][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-26 21:57:03,264][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-26 21:57:03,268][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-26 21:57:03,269][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-26 21:57:03,273][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-26 21:57:03,286][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-26 21:57:03,287][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-26 21:57:03,295][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-26 21:57:08,860][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-26 21:57:28,216][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:57:48,647][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,651][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,657][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,663][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,668][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,670][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,754][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:48,808][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 21:57:50,706][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 21:57:51,793][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:52,794][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 21:57:54,645][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 21:58:17,968][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 21:58:20,252][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 21:58:20,253][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 21:58:20,253][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 21:58:20,253][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 21:58:20,253][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 22:00:06,501][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 22:00:28,913][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,917][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,924][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,929][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,932][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,933][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,936][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:28,937][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 22:00:29,725][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 22:00:30,865][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:00:30,893][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 22:03:14,646][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:03:14,660][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:03:14,660][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 22:03:14,660][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-28 15:38:08,155][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=200000` reached. diff --git a/outputs/2026-06-26/23-03-39/.hydra/config.yaml b/outputs/2026-06-26/23-03-39/.hydra/config.yaml deleted file mode 100644 index 3e0d1eacb4a756bcfda7d47203d6772fdda4254a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-39/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: false - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-26/23-03-39/.hydra/hydra.yaml b/outputs/2026-06-26/23-03-39/.hydra/hydra.yaml deleted file mode 100644 index 99bd6011eb8e1d54ff499c9af5654c318fe705d0..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-39/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-26/23-03-39 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-26/23-03-39/.hydra/overrides.yaml b/outputs/2026-06-26/23-03-39/.hydra/overrides.yaml deleted file mode 100644 index 5c8056a58520299ba8204bad4bbcc640e272ce00..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-39/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-26/23-03-39/main.log b/outputs/2026-06-26/23-03-39/main.log deleted file mode 100644 index cfb455ec88a48f8f76ff1d5cd0cc83fc2ba3a6b8..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-39/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-26 23:05:24,503][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-26 23:06:11,327][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:33,428][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-26/23-03-40/.hydra/config.yaml b/outputs/2026-06-26/23-03-40/.hydra/config.yaml deleted file mode 100644 index 3e0d1eacb4a756bcfda7d47203d6772fdda4254a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-40/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: false - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-26/23-03-40/.hydra/hydra.yaml b/outputs/2026-06-26/23-03-40/.hydra/hydra.yaml deleted file mode 100644 index 78be1f76e6f591b414b5ad4646fd03f4f830b31e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-40/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-26/23-03-40 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-26/23-03-40/.hydra/overrides.yaml b/outputs/2026-06-26/23-03-40/.hydra/overrides.yaml deleted file mode 100644 index 5c8056a58520299ba8204bad4bbcc640e272ce00..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-40/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-26/23-03-40/main.log b/outputs/2026-06-26/23-03-40/main.log deleted file mode 100644 index ac3c1955fe34eb353ad8b2b6b11339ea7c2de678..0000000000000000000000000000000000000000 --- a/outputs/2026-06-26/23-03-40/main.log +++ /dev/null @@ -1,88 +0,0 @@ -[2026-06-26 23:04:33,447][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 23:04:33,664][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 23:04:33,664][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 23:04:33,664][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 23:04:33,664][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 23:04:33,665][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 23:05:24,505][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-26 23:05:24,507][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-26 23:05:24,518][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-26 23:05:24,527][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-26 23:05:24,529][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-26 23:05:24,539][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-26 23:05:24,543][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-26 23:05:30,498][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-26 23:05:49,112][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 23:06:11,289][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,308][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,320][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,341][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,346][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,348][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:11,382][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:06:13,352][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 23:06:14,390][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:15,383][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:06:16,988][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-26 23:06:40,779][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-26 23:06:41,977][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-26 23:06:41,978][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-26 23:06:41,978][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-26 23:06:41,978][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-26 23:06:41,978][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-26 23:08:27,478][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 23:08:33,430][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:33,611][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:33,612][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:33,613][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:33,616][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:33,694][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:34,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-26 23:08:34,736][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-26 23:08:35,995][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_memory_noise_a2d4r2_route_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:08:36,001][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-26 23:11:13,621][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 23:11:13,633][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 23:11:13,633][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-26 23:11:13,633][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-28 16:32:53,866][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=200000` reached. diff --git a/outputs/2026-06-27/02-41-42/.hydra/config.yaml b/outputs/2026-06-27/02-41-42/.hydra/config.yaml deleted file mode 100644 index d24738b99d91e3ae14cbbe8a4c211bcf4b2fc76b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-42/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-27/02-41-42/.hydra/hydra.yaml b/outputs/2026-06-27/02-41-42/.hydra/hydra.yaml deleted file mode 100644 index 38441838abbd89c2506a29ad3c6bd51cc3f0f514..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-42/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-27/02-41-42 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-27/02-41-42/.hydra/overrides.yaml b/outputs/2026-06-27/02-41-42/.hydra/overrides.yaml deleted file mode 100644 index 4b85025abd8fab687599f0ac1ed0ad4c8ddb9f9a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-42/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-27/02-41-42/main.log b/outputs/2026-06-27/02-41-42/main.log deleted file mode 100644 index cb3acb8703a578c3df3e2cea30030bcbdd7b4fa5..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-42/main.log +++ /dev/null @@ -1,5 +0,0 @@ -[2026-06-27 02:43:17,623][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-27 02:44:04,956][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:44,479][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:46,423][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-27/02-41-43/.hydra/config.yaml b/outputs/2026-06-27/02-41-43/.hydra/config.yaml deleted file mode 100644 index d24738b99d91e3ae14cbbe8a4c211bcf4b2fc76b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-43/.hydra/config.yaml +++ /dev/null @@ -1,211 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 5000 - every_n_epochs: null - train_time_interval: null - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 60000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 5000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 700 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 5000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 20000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-27/02-41-43/.hydra/hydra.yaml b/outputs/2026-06-27/02-41-43/.hydra/hydra.yaml deleted file mode 100644 index 147c657e131c904cf58b8f5549101a7a9630614e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-43/.hydra/hydra.yaml +++ /dev/null @@ -1,216 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=700 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=5000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=20000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=5000 - - experiment.validation.val_every_n_step=5000 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=20000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=5000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=700,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=5000,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=5000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-27/02-41-43 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-27/02-41-43/.hydra/overrides.yaml b/outputs/2026-06-27/02-41-43/.hydra/overrides.yaml deleted file mode 100644 index 4b85025abd8fab687599f0ac1ed0ad4c8ddb9f9a..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-43/.hydra/overrides.yaml +++ /dev/null @@ -1,58 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=700 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=5000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=20000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:60000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=5000 -- experiment.validation.val_every_n_step=5000 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-06-27/02-41-43/main.log b/outputs/2026-06-27/02-41-43/main.log deleted file mode 100644 index 9018c8fd5d634eee9192b2d811f1db72ab3bfc45..0000000000000000000000000000000000000000 --- a/outputs/2026-06-27/02-41-43/main.log +++ /dev/null @@ -1,87 +0,0 @@ -[2026-06-27 02:42:28,496][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-27 02:42:28,648][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-27 02:42:28,649][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-27 02:42:28,649][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-27 02:42:28,649][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-27 02:42:28,649][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-27 02:43:17,576][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-27 02:43:17,577][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-27 02:43:17,593][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-27 02:43:17,601][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-27 02:43:17,612][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-27 02:43:17,614][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-27 02:43:17,621][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-27 02:43:23,456][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-27 02:43:42,740][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-27 02:44:04,944][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,949][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,951][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,951][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,952][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,954][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:04,958][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:44:06,715][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-27 02:44:07,776][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:09,717][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:44:11,952][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=60000` reached. -[2026-06-27 02:44:34,170][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-27 02:44:34,986][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-27 02:44:34,986][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-27 02:44:34,986][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-27 02:44:34,986][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-27 02:44:34,986][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-27 02:46:21,004][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-27 02:46:44,465][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,466][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,469][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,470][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,471][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,472][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:44,474][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-27 02:46:45,267][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-27 02:46:46,411][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_a2d4r2_ntok8_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_step60000.ckpt -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:46:46,422][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-27 02:49:25,621][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-27 02:49:26,026][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-27 02:49:26,027][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-27 02:49:26,027][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - diff --git a/outputs/2026-06-30/19-06-53/.hydra/config.yaml b/outputs/2026-06-30/19-06-53/.hydra/config.yaml deleted file mode 100644 index c81ef2c4af5ee421ba6f3a9526a388b050457bab..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-53/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-06-53/.hydra/hydra.yaml b/outputs/2026-06-30/19-06-53/.hydra/hydra.yaml deleted file mode 100644 index 4e075c4e1c6082fa626b19bb8159fccc31817528..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-53/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-06-53 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-06-53/.hydra/overrides.yaml b/outputs/2026-06-30/19-06-53/.hydra/overrides.yaml deleted file mode 100644 index 848c831fb4d100b4cb3423a74c6f383fd35a56fb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-53/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-06-53/main.log b/outputs/2026-06-30/19-06-53/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-06-54/.hydra/config.yaml b/outputs/2026-06-30/19-06-54/.hydra/config.yaml deleted file mode 100644 index c81ef2c4af5ee421ba6f3a9526a388b050457bab..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-54/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-06-54/.hydra/hydra.yaml b/outputs/2026-06-30/19-06-54/.hydra/hydra.yaml deleted file mode 100644 index fbff5f53736958c46391aa3e9fbd37e586f14d8c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-54/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-06-54 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-06-54/.hydra/overrides.yaml b/outputs/2026-06-30/19-06-54/.hydra/overrides.yaml deleted file mode 100644 index 848c831fb4d100b4cb3423a74c6f383fd35a56fb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-54/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-06-54/main.log b/outputs/2026-06-30/19-06-54/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-06-55/.hydra/config.yaml b/outputs/2026-06-30/19-06-55/.hydra/config.yaml deleted file mode 100644 index c81ef2c4af5ee421ba6f3a9526a388b050457bab..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-55/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-06-55/.hydra/hydra.yaml b/outputs/2026-06-30/19-06-55/.hydra/hydra.yaml deleted file mode 100644 index a2e872d614bcbfb9dac5e75ba0060d58c51edc64..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-55/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-06-55 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-06-55/.hydra/overrides.yaml b/outputs/2026-06-30/19-06-55/.hydra/overrides.yaml deleted file mode 100644 index 848c831fb4d100b4cb3423a74c6f383fd35a56fb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-55/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-06-55/main.log b/outputs/2026-06-30/19-06-55/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-06-59/.hydra/config.yaml b/outputs/2026-06-30/19-06-59/.hydra/config.yaml deleted file mode 100644 index 44622f84726ec574fca73aa589ee22826a9d62a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-59/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: false - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-06-59/.hydra/hydra.yaml b/outputs/2026-06-30/19-06-59/.hydra/hydra.yaml deleted file mode 100644 index 52889d5ce0492fce5302e227f4e2834f47b46a45..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-59/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=false - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=false,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-06-59 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-06-59/.hydra/overrides.yaml b/outputs/2026-06-30/19-06-59/.hydra/overrides.yaml deleted file mode 100644 index efdd985de2643ea464ba05109c047aedfa2b3a80..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-06-59/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=false -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-06-59/main.log b/outputs/2026-06-30/19-06-59/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-07-00/.hydra/config.yaml b/outputs/2026-06-30/19-07-00/.hydra/config.yaml deleted file mode 100644 index 44622f84726ec574fca73aa589ee22826a9d62a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-00/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: false - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-07-00/.hydra/hydra.yaml b/outputs/2026-06-30/19-07-00/.hydra/hydra.yaml deleted file mode 100644 index d8eb191b6cad90e142aa8898e21446de2e37b1cf..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-00/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=false - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=false,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-07-00 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-07-00/.hydra/overrides.yaml b/outputs/2026-06-30/19-07-00/.hydra/overrides.yaml deleted file mode 100644 index efdd985de2643ea464ba05109c047aedfa2b3a80..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-00/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=false -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-07-00/main.log b/outputs/2026-06-30/19-07-00/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-07-07/.hydra/config.yaml b/outputs/2026-06-30/19-07-07/.hydra/config.yaml deleted file mode 100644 index 8ff5d963c0848b9bb2ea4f52dadab7f3ccf650b6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-07/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-07-07/.hydra/hydra.yaml b/outputs/2026-06-30/19-07-07/.hydra/hydra.yaml deleted file mode 100644 index 71a9e21e21314903c497f5ca1693847b81cf9c57..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-07/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-07-07 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-07-07/.hydra/overrides.yaml b/outputs/2026-06-30/19-07-07/.hydra/overrides.yaml deleted file mode 100644 index 7e3f2353d03287af5ccbd829323aa8d3239cc6e1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-07-07/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-07-07/main.log b/outputs/2026-06-30/19-07-07/main.log deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/outputs/2026-06-30/19-23-38/.hydra/config.yaml b/outputs/2026-06-30/19-23-38/.hydra/config.yaml deleted file mode 100644 index c81ef2c4af5ee421ba6f3a9526a388b050457bab..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-38/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-38/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-38/.hydra/hydra.yaml deleted file mode 100644 index 7cb9ab94762f643ca12f7da3bb6ccc96cfd46c2c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-38/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-38 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-38/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-38/.hydra/overrides.yaml deleted file mode 100644 index 848c831fb4d100b4cb3423a74c6f383fd35a56fb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-38/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-38/main.log b/outputs/2026-06-30/19-23-38/main.log deleted file mode 100644 index 4be82153d1cd1e929fe2bb9083142f9f2b4408b9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-38/main.log +++ /dev/null @@ -1,155 +0,0 @@ -[2026-06-30 19:24:28,849][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:28,850][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-30 19:24:28,939][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-30 19:24:28,939][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-30 19:24:29,025][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:29,062][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-30 19:24:29,107][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-30 19:24:29,256][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-30 19:24:29,257][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-30 19:24:29,257][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-30 19:24:29,257][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-30 19:24:29,257][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-30 19:25:02,079][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-30 19:25:02,079][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-30 19:25:02,079][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-30 19:25:02,080][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-30 19:25:02,183][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-30 19:25:02,780][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-30 19:25:08,639][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:17,764][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 20:07:43,008][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,150][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:07:56,152][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-01 06:32:04,534][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=30000` reached. -[2026-07-01 06:32:26,101][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,618][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-01 06:32:26,618][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-01 06:32:26,618][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-01 06:32:26,618][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-01 06:32:26,619][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-01 06:34:09,649][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints exists and is not empty. - -[2026-07-01 06:34:09,649][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:34:09,660][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,662][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,664][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,667][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,670][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,673][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:15,865][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,198][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,198][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,273][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,464][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,569][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:17,371][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-07-01 06:34:18,560][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:19,846][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-03 05:19:16,315][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=150000` reached. diff --git a/outputs/2026-06-30/19-23-39/.hydra/config.yaml b/outputs/2026-06-30/19-23-39/.hydra/config.yaml deleted file mode 100644 index c81ef2c4af5ee421ba6f3a9526a388b050457bab..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-39/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-39/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-39/.hydra/hydra.yaml deleted file mode 100644 index cafbf07f8f5322dd6f394dc43d87257559ab074e..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-39/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-39 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-39/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-39/.hydra/overrides.yaml deleted file mode 100644 index 848c831fb4d100b4cb3423a74c6f383fd35a56fb..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-39/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-39/main.log b/outputs/2026-06-30/19-23-39/main.log deleted file mode 100644 index afbae609ba738f868915291cef99a4cd16618636..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-39/main.log +++ /dev/null @@ -1,18 +0,0 @@ -[2026-06-30 19:25:02,080][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-30 19:25:02,080][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:14,146][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:17,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:32:26,102][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:34:09,668][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:09,670][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:34:15,975][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:16,467][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:34:18,606][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-30/19-23-44/.hydra/config.yaml b/outputs/2026-06-30/19-23-44/.hydra/config.yaml deleted file mode 100644 index 44622f84726ec574fca73aa589ee22826a9d62a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-44/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: false - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-44/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-44/.hydra/hydra.yaml deleted file mode 100644 index ecb036598152760cb4e0ee17aafbe9c0ac36f726..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-44/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=false - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=false,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-44 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-44/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-44/.hydra/overrides.yaml deleted file mode 100644 index efdd985de2643ea464ba05109c047aedfa2b3a80..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-44/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=false -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-44/main.log b/outputs/2026-06-30/19-23-44/main.log deleted file mode 100644 index c0cb114a5f76aab0dd2bd8a76d01a867040f15ae..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-44/main.log +++ /dev/null @@ -1,29 +0,0 @@ -[2026-06-30 19:25:10,901][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-30 19:25:10,903][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-30 19:25:10,904][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:33:24,811][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:35:15,565][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:15,578][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:15,580][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:34,844][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:34,846][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:34,858][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 23:51:08,644][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-07-01 23:51:08,645][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-30/19-23-45/.hydra/config.yaml b/outputs/2026-06-30/19-23-45/.hydra/config.yaml deleted file mode 100644 index 44622f84726ec574fca73aa589ee22826a9d62a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-45/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: false - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-45/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-45/.hydra/hydra.yaml deleted file mode 100644 index af7cbc1c62907cf1ce5f6625049f0d06d8b21f4c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-45/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=false - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=false,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-45 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-45/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-45/.hydra/overrides.yaml deleted file mode 100644 index efdd985de2643ea464ba05109c047aedfa2b3a80..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-45/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=false -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-45/main.log b/outputs/2026-06-30/19-23-45/main.log deleted file mode 100644 index fb118e548e7a958d9c864d0be3994cbe8c993c72..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-45/main.log +++ /dev/null @@ -1,144 +0,0 @@ -[2026-06-30 19:24:35,855][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:35,856][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-30 19:24:35,949][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-30 19:24:35,950][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-30 19:24:36,037][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:36,075][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-30 19:24:36,120][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-30 19:24:36,269][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-30 19:24:36,270][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-30 19:24:36,270][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-30 19:24:36,270][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-30 19:24:36,270][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-30 19:25:10,900][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-30 19:25:10,900][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-30 19:25:11,204][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-30 19:25:11,273][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-30 19:25:17,651][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:24,713][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-30 19:25:25,768][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 20:08:00,782][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,367][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,368][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,369][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:08:03,370][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-01 06:33:03,044][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=30000` reached. -[2026-07-01 06:33:24,811][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:24,813][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:33:25,110][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-01 06:33:25,110][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-01 06:33:25,110][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-01 06:33:25,110][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-01 06:33:25,111][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-01 06:35:15,557][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints exists and is not empty. - -[2026-07-01 06:35:15,558][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:35:15,572][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:15,573][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:15,575][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:15,582][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:34,844][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:34,850][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:34,852][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:34,855][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:35,707][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-07-01 06:35:36,832][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:35:38,048][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-01 23:51:08,642][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-07-01 23:51:08,642][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-07-01 23:51:08,643][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-01 23:51:08,643][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-01 23:51:08,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-07-01 23:51:08,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-01 23:51:08,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-07-01 23:51:08,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-30/19-23-46/.hydra/config.yaml b/outputs/2026-06-30/19-23-46/.hydra/config.yaml deleted file mode 100644 index 44622f84726ec574fca73aa589ee22826a9d62a2..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-46/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: false - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-46/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-46/.hydra/hydra.yaml deleted file mode 100644 index 3aa259464e34b44a8d149c297e9fe78a4896bb0c..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-46/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=false - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=false,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-46 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-46/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-46/.hydra/overrides.yaml deleted file mode 100644 index efdd985de2643ea464ba05109c047aedfa2b3a80..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-46/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=false -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-46/main.log b/outputs/2026-06-30/19-23-46/main.log deleted file mode 100644 index 86a073a2a85061ee06a23e8f2503bd9e9b1b697b..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-46/main.log +++ /dev/null @@ -1,11 +0,0 @@ -[2026-06-30 19:25:10,904][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-30 19:25:23,015][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:25,769][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:33:24,812][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:35:15,583][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:35:34,844][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:35:36,897][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 23:51:08,649][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-07-01 23:51:08,650][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-06-30/19-23-51/.hydra/config.yaml b/outputs/2026-06-30/19-23-51/.hydra/config.yaml deleted file mode 100644 index 8ff5d963c0848b9bb2ea4f52dadab7f3ccf650b6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-51/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-51/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-51/.hydra/hydra.yaml deleted file mode 100644 index 7d5f60ec77d8f901fece75bd729afe31cf35aae9..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-51/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-51 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-51/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-51/.hydra/overrides.yaml deleted file mode 100644 index 7e3f2353d03287af5ccbd829323aa8d3239cc6e1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-51/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-51/main.log b/outputs/2026-06-30/19-23-51/main.log deleted file mode 100644 index 505ec37d440805746ae02faca39eef77f8e31917..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-51/main.log +++ /dev/null @@ -1,54 +0,0 @@ -[2026-06-30 19:25:17,363][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-06-30 19:25:17,364][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-06-30 19:25:17,366][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-06-30 19:25:17,366][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-06-30 19:25:17,367][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-06-30 19:25:17,368][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:58:41,444][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,448][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,449][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,452][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,456][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,457][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:59:01,169][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,171][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,176][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,190][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,191][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,192][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-06-30/19-23-52/.hydra/config.yaml b/outputs/2026-06-30/19-23-52/.hydra/config.yaml deleted file mode 100644 index 8ff5d963c0848b9bb2ea4f52dadab7f3ccf650b6..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-52/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 150000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 10 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 150000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.25 - revisit_max_fraction: 0.25 - noise_route: - anchor: all - dynamic: all - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-06-30/19-23-52/.hydra/hydra.yaml b/outputs/2026-06-30/19-23-52/.hydra/hydra.yaml deleted file mode 100644 index 6f047fd30b78cc1da0300e3837b19fec0ad404e7..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-52/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=all - - algorithm.noise_route.dynamic=all - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.25 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=10 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=150000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.25,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=all,algorithm.noise_route.dynamic=all,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=10,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]},experiment.training.max_steps=150000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-06-30/19-23-52 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-06-30/19-23-52/.hydra/overrides.yaml b/outputs/2026-06-30/19-23-52/.hydra/overrides.yaml deleted file mode 100644 index 7e3f2353d03287af5ccbd829323aa8d3239cc6e1..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-52/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=all -- algorithm.noise_route.dynamic=all -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.25 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:150000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=10 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=150000 diff --git a/outputs/2026-06-30/19-23-52/main.log b/outputs/2026-06-30/19-23-52/main.log deleted file mode 100644 index 7034a9f7955637cca654c1f04012053c267d497d..0000000000000000000000000000000000000000 --- a/outputs/2026-06-30/19-23-52/main.log +++ /dev/null @@ -1,119 +0,0 @@ -[2026-06-30 19:24:40,658][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:40,659][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-06-30 19:24:40,754][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-06-30 19:24:40,754][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-06-30 19:24:40,838][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-06-30 19:24:40,876][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-06-30 19:24:40,920][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-06-30 19:24:41,066][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-06-30 19:24:41,066][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-06-30 19:24:41,066][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-06-30 19:24:41,066][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-06-30 19:24:41,067][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-06-30 19:25:17,365][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-06-30 19:25:17,740][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-06-30 19:25:24,027][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:29,276][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-06-30 19:25:32,435][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-06-30 19:25:32,436][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 19:25:32,437][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-06-30 20:09:28,129][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,297][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,297][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,297][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,298][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,299][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-06-30 20:09:43,300][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-01 06:56:34,634][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=30000` reached. -[2026-07-01 06:56:56,994][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-01 06:56:56,994][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:56,995][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-01 06:56:58,603][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-01 06:56:58,603][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-01 06:56:58,604][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-01 06:56:58,604][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-01 06:56:58,605][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-01 06:58:41,430][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/checkpoints exists and is not empty. - -[2026-07-01 06:58:41,431][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:58:41,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:58:41,454][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-01 06:59:01,193][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:01,245][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-01 06:59:02,086][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-07-01 06:59:03,240][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:03,260][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-01 06:59:04,353][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-03 07:28:34,306][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=150000` reached. diff --git a/outputs/2026-07-03/18-31-12/.hydra/config.yaml b/outputs/2026-07-03/18-31-12/.hydra/config.yaml deleted file mode 100644 index 4ba66d1cd911a8035a8bc6462ae3b82c497859b1..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-12/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.0 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-03/18-31-12/.hydra/hydra.yaml b/outputs/2026-07-03/18-31-12/.hydra/hydra.yaml deleted file mode 100644 index 9548a87efcee3f7aab8dd96a51cb21db59d32309..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-12/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=0 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.0 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.0,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=0,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-03/18-31-12 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-03/18-31-12/.hydra/overrides.yaml b/outputs/2026-07-03/18-31-12/.hydra/overrides.yaml deleted file mode 100644 index a11b1f8dea651b89d720494b95235887b8322312..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-12/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=0 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.0 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-03/18-31-12/main.log b/outputs/2026-07-03/18-31-12/main.log deleted file mode 100644 index 8f126fd377588eebac9a8f0a692ca4a3eee90eb9..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-12/main.log +++ /dev/null @@ -1,9 +0,0 @@ -[2026-07-03 18:33:09,225][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:10:25,149][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:45,413][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-03/18-31-13/.hydra/config.yaml b/outputs/2026-07-03/18-31-13/.hydra/config.yaml deleted file mode 100644 index 4ba66d1cd911a8035a8bc6462ae3b82c497859b1..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-13/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.0 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-03/18-31-13/.hydra/hydra.yaml b/outputs/2026-07-03/18-31-13/.hydra/hydra.yaml deleted file mode 100644 index 6a9a01a4185288d7bf52556d3ede1f6956f901da..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-13/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=0 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.0 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.0,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=0,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-03/18-31-13 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-03/18-31-13/.hydra/overrides.yaml b/outputs/2026-07-03/18-31-13/.hydra/overrides.yaml deleted file mode 100644 index a11b1f8dea651b89d720494b95235887b8322312..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-13/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=0 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.0 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-03/18-31-13/main.log b/outputs/2026-07-03/18-31-13/main.log deleted file mode 100644 index 259a67716c02623e545a2c318b845d957577fbc0..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-13/main.log +++ /dev/null @@ -1,9 +0,0 @@ -[2026-07-03 18:33:09,226][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:08:28,448][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:10:25,144][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:45,412][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-03/18-31-16/.hydra/config.yaml b/outputs/2026-07-03/18-31-16/.hydra/config.yaml deleted file mode 100644 index 4ba66d1cd911a8035a8bc6462ae3b82c497859b1..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-16/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.0 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-03/18-31-16/.hydra/hydra.yaml b/outputs/2026-07-03/18-31-16/.hydra/hydra.yaml deleted file mode 100644 index c59c7d19cb0e6413c55d3caaee7b12ab96bdd96d..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-16/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=0 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.0 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.0,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=0,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-03/18-31-16 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-03/18-31-16/.hydra/overrides.yaml b/outputs/2026-07-03/18-31-16/.hydra/overrides.yaml deleted file mode 100644 index a11b1f8dea651b89d720494b95235887b8322312..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-16/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=0 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.0 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-03/18-31-16/main.log b/outputs/2026-07-03/18-31-16/main.log deleted file mode 100644 index e498b55af1d3a408c410704a800afee09bf5b5ad..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-16/main.log +++ /dev/null @@ -1,18 +0,0 @@ -[2026-07-03 18:33:09,228][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-03 18:33:09,492][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:10:25,142][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:25,147][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:45,414][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:45,418][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-03/18-31-17/.hydra/config.yaml b/outputs/2026-07-03/18-31-17/.hydra/config.yaml deleted file mode 100644 index 4ba66d1cd911a8035a8bc6462ae3b82c497859b1..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-17/.hydra/config.yaml +++ /dev/null @@ -1,214 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 30000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 0 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - dynamic: - selection_policy: event_triggered - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - pose_preselect_topk: 64 - candidate_chunk_size: 64 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 1000 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.0 - dynamic_max_fraction: 0.0 - revisit_max_fraction: 0.25 - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - add_timestamp_embedding: true - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: true - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-03/18-31-17/.hydra/hydra.yaml b/outputs/2026-07-03/18-31-17/.hydra/hydra.yaml deleted file mode 100644 index 81c727304641d159a19a1061efb6bd71ff155ace..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-17/.hydra/hydra.yaml +++ /dev/null @@ -1,219 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=0 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.pose_similarity_threshold=0.6 - - dataset.memory_selection.training_use_plucker=true - - dataset.memory_selection.training_plucker_weight=1.0 - - dataset.memory_selection.fov_overlap_threshold=0.6 - - dataset.memory_selection.min_total_selected_coverage=0.1 - - dataset.memory_selection.local_context_exclusion_frames=8 - - dataset.memory_selection.anchor_diverse_selection=true - - +dataset.memory_selection.pose_preselect_topk=64 - - +dataset.memory_selection.candidate_chunk_size=64 - - dataset.memory_selection.dynamic.selection_policy=event_triggered - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.chunk_size=1 - - algorithm.warmup_steps=1000 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=true - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.0 - - algorithm.memory_noise.dynamic_max_fraction=0.0 - - algorithm.memory_noise.revisit_max_fraction=0.25 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+dataset.memory_selection.candidate_chunk_size=64,+dataset.memory_selection.pose_preselect_topk=64,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_noise.anchor_max_fraction=0.0,algorithm.memory_noise.dynamic_max_fraction=0.0,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.25,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=true,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=1000,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.anchor_diverse_selection=true,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.selection_policy=event_triggered,dataset.memory_selection.enabled=true,dataset.memory_selection.fov_overlap_threshold=0.6,dataset.memory_selection.local_context_exclusion_frames=8,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=0,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.min_total_selected_coverage=0.1,dataset.memory_selection.pose_similarity_threshold=0.6,dataset.memory_selection.training_plucker_weight=1.0,dataset.memory_selection.training_use_plucker=true,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-03/18-31-17 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-03/18-31-17/.hydra/overrides.yaml b/outputs/2026-07-03/18-31-17/.hydra/overrides.yaml deleted file mode 100644 index a11b1f8dea651b89d720494b95235887b8322312..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-17/.hydra/overrides.yaml +++ /dev/null @@ -1,61 +0,0 @@ -- +name=train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=0 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.pose_similarity_threshold=0.6 -- dataset.memory_selection.training_use_plucker=true -- dataset.memory_selection.training_plucker_weight=1.0 -- dataset.memory_selection.fov_overlap_threshold=0.6 -- dataset.memory_selection.min_total_selected_coverage=0.1 -- dataset.memory_selection.local_context_exclusion_frames=8 -- dataset.memory_selection.anchor_diverse_selection=true -- +dataset.memory_selection.pose_preselect_topk=64 -- +dataset.memory_selection.candidate_chunk_size=64 -- dataset.memory_selection.dynamic.selection_policy=event_triggered -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.chunk_size=1 -- algorithm.warmup_steps=1000 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=true -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.0 -- algorithm.memory_noise.dynamic_max_fraction=0.0 -- algorithm.memory_noise.revisit_max_fraction=0.25 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:30000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-03/18-31-17/main.log b/outputs/2026-07-03/18-31-17/main.log deleted file mode 100644 index f3466edeed9d6a0fc56459c3d3772a484d05646a..0000000000000000000000000000000000000000 --- a/outputs/2026-07-03/18-31-17/main.log +++ /dev/null @@ -1,137 +0,0 @@ -[2026-07-03 18:32:25,228][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-03 18:32:25,229][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: timestamp_embedding.mlp.0.weight, timestamp_embedding.mlp.0.bias, timestamp_embedding.mlp.2.weight, timestamp_embedding.mlp.2.bias, spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, ... (+560 more) -[2026-07-03 18:32:25,318][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-03 18:32:25,319][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-03 18:32:25,386][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-03 18:32:25,417][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-03 18:32:25,458][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-03 18:32:25,688][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-03 18:32:25,688][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-03 18:32:25,688][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-03 18:32:25,688][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-03 18:32:25,688][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-03 18:33:09,225][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-03 18:33:09,227][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-03 18:33:09,228][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-03 18:33:09,850][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-03 18:33:15,942][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:20,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-03 18:33:22,391][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-03 18:33:23,499][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-03 19:06:03,246][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,990][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,990][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,990][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,990][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,991][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-03 19:06:07,992][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-04 03:08:07,035][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=30000` reached. -[2026-07-04 03:08:28,447][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:08:28,447][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:08:28,448][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-04 03:08:29,459][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-04 03:08:29,460][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-04 03:08:29,460][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-04 03:08:29,460][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-04 03:08:29,461][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-04 03:10:25,131][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/checkpoints exists and is not empty. - -[2026-07-04 03:10:25,132][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-04 03:10:25,139][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:25,142][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:25,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:25,151][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-04 03:10:45,416][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:45,417][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:45,419][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:45,419][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-04 03:10:46,226][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -438 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,112.927 Total estimated model params size (MB) -[2026-07-04 03:10:47,297][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step30000.ckpt -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:10:47,312][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-04 03:10:48,296][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-05 06:27:45,563][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=120000` reached. diff --git a/outputs/2026-07-06/21-35-50/.hydra/config.yaml b/outputs/2026-07-06/21-35-50/.hydra/config.yaml deleted file mode 100644 index 024cbcfd2e5d40b89d32122ae2898bbb2bfdd2f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-35-50/.hydra/config.yaml +++ /dev/null @@ -1,218 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-06/21-35-50/.hydra/hydra.yaml b/outputs/2026-07-06/21-35-50/.hydra/hydra.yaml deleted file mode 100644 index f854ccc81128776a515d6162d63fb82263def609..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-35-50/.hydra/hydra.yaml +++ /dev/null @@ -1,212 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.50 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.50,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-06/21-35-50 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-06/21-35-50/.hydra/overrides.yaml b/outputs/2026-07-06/21-35-50/.hydra/overrides.yaml deleted file mode 100644 index b30d277b6349296325e79fe83f661bca8f3765a3..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-35-50/.hydra/overrides.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- +name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.50 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-06/21-35-50/main.log b/outputs/2026-07-06/21-35-50/main.log deleted file mode 100644 index 3facc78b7788c9efde43f0113837d9057646cf59..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-35-50/main.log +++ /dev/null @@ -1,172 +0,0 @@ -[2026-07-06 21:36:46,882][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-06 21:36:46,883][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, blocks.0.r_mlp.fc1.weight, blocks.0.r_mlp.fc1.bias, ... (+556 more) -[2026-07-06 21:36:46,973][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-06 21:36:46,974][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-06 21:36:47,074][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-06 21:36:47,106][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-06 21:36:47,167][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-06 21:36:47,375][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-06 21:36:47,375][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-06 21:36:47,375][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-06 21:36:47,375][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-06 21:36:47,376][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-06 21:37:17,625][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-06 21:37:17,625][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-06 21:37:17,625][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-06 21:37:17,628][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-06 21:37:17,630][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-06 21:37:17,637][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-06 21:37:17,789][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-06 21:37:17,828][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-06 21:37:23,966][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:29,128][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:37:30,871][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -437 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,107.676 Total estimated model params size (MB) -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:37:33,445][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 22:18:24,382][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,168][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,169][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:18:26,170][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-07 11:56:24,926][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,313][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,313][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,314][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:46,314][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:56:47,376][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-07 11:56:47,376][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-07 11:56:47,376][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-07 11:56:47,376][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-07 11:56:47,377][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-07 11:58:28,388][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints exists and is not empty. - -[2026-07-07 11:58:28,390][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step40000.ckpt -[2026-07-07 11:58:28,396][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,396][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,402][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,403][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,405][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,407][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,410][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:28,412][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:58:51,356][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,356][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,358][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,359][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,359][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,360][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,364][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:51,365][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:58:52,200][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -437 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,107.676 Total estimated model params size (MB) -[2026-07-07 11:58:53,323][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step40000.ckpt -[2026-07-07 11:58:53,349][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,349][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,350][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,350][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,350][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,349][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,350][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:53,350][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:58:54,450][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-07-06/21-51-21/.hydra/config.yaml b/outputs/2026-07-06/21-51-21/.hydra/config.yaml deleted file mode 100644 index da455977ca073cd59b672c4c743d3c545a8898f6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-21/.hydra/config.yaml +++ /dev/null @@ -1,225 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-06/21-51-21/.hydra/hydra.yaml b/outputs/2026-07-06/21-51-21/.hydra/hydra.yaml deleted file mode 100644 index eb355f507f4e625c592a41498bbeb4b1823b7420..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-21/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-06/21-51-21 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-06/21-51-21/.hydra/overrides.yaml b/outputs/2026-07-06/21-51-21/.hydra/overrides.yaml deleted file mode 100644 index f64ce23a93228172d6a76c751896c3e0f6141614..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-21/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-06/21-51-21/main.log b/outputs/2026-07-06/21-51-21/main.log deleted file mode 100644 index 30febcb0a2e0dca0e43ea64beda86b69e90152b7..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-21/main.log +++ /dev/null @@ -1,117 +0,0 @@ -[2026-07-06 21:52:26,797][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-06 21:52:26,798][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, blocks.0.r_mlp.fc1.weight, blocks.0.r_mlp.fc1.bias, ... (+300 more) -[2026-07-06 21:52:26,886][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-06 21:52:26,887][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-06 21:52:26,966][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-06 21:52:26,998][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-06 21:52:27,038][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-06 21:52:27,185][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-06 21:52:27,185][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-06 21:52:27,185][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-06 21:52:27,185][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-06 21:52:27,186][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-06 21:53:04,974][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-06 21:53:04,978][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-06 21:53:04,980][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-06 21:53:10,508][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:17,196][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:53:19,135][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 22:31:44,044][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,195][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,195][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,196][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-06 22:31:50,197][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-07 11:23:58,728][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-07 11:24:18,527][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:18,528][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:19,875][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-07 11:24:19,875][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-07 11:24:19,875][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-07 11:24:19,875][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-07 11:24:19,876][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-07 11:26:08,907][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/checkpoints exists and is not empty. - -[2026-07-07 11:26:08,908][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step40000.ckpt -[2026-07-07 11:26:08,914][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:08,915][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:08,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:27,138][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,147][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,152][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,963][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-07 11:26:28,925][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/checkpoints/curriculum_01_no_updown_step40000.ckpt -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:29,819][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - diff --git a/outputs/2026-07-06/21-51-22/.hydra/config.yaml b/outputs/2026-07-06/21-51-22/.hydra/config.yaml deleted file mode 100644 index da455977ca073cd59b672c4c743d3c545a8898f6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-22/.hydra/config.yaml +++ /dev/null @@ -1,225 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-06/21-51-22/.hydra/hydra.yaml b/outputs/2026-07-06/21-51-22/.hydra/hydra.yaml deleted file mode 100644 index 6ebcaef60392bb4cf77f28af6f5810653d7fde13..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-22/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-06/21-51-22 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-06/21-51-22/.hydra/overrides.yaml b/outputs/2026-07-06/21-51-22/.hydra/overrides.yaml deleted file mode 100644 index f64ce23a93228172d6a76c751896c3e0f6141614..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-22/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-06/21-51-22/main.log b/outputs/2026-07-06/21-51-22/main.log deleted file mode 100644 index 23d3aa6bf2a72545d2dbe3aeac0af89bd7fa2d34..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-22/main.log +++ /dev/null @@ -1,36 +0,0 @@ -[2026-07-06 21:53:04,975][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-06 21:53:04,976][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-06 21:53:04,978][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-06 21:53:04,983][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:26:08,916][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:08,917][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:08,925][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:08,926][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:27,137][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,141][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,147][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:27,152][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-06/21-51-23/.hydra/config.yaml b/outputs/2026-07-06/21-51-23/.hydra/config.yaml deleted file mode 100644 index da455977ca073cd59b672c4c743d3c545a8898f6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-23/.hydra/config.yaml +++ /dev/null @@ -1,225 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-06/21-51-23/.hydra/hydra.yaml b/outputs/2026-07-06/21-51-23/.hydra/hydra.yaml deleted file mode 100644 index 57bbe4f3dcef779e5d3ff1bd6e495aae7ad26e5a..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-23/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-06/21-51-23 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-06/21-51-23/.hydra/overrides.yaml b/outputs/2026-07-06/21-51-23/.hydra/overrides.yaml deleted file mode 100644 index f64ce23a93228172d6a76c751896c3e0f6141614..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-23/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-06/21-51-23/main.log b/outputs/2026-07-06/21-51-23/main.log deleted file mode 100644 index ce90917957fbba4b7d9e9da8d44efdef4cff4393..0000000000000000000000000000000000000000 --- a/outputs/2026-07-06/21-51-23/main.log +++ /dev/null @@ -1,9 +0,0 @@ -[2026-07-06 21:53:04,976][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-06 21:53:16,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-06 21:53:19,136][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-07 11:24:18,527][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-07 11:26:08,926][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-07 11:26:27,281][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-07 11:26:28,942][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-10/11-07-57/.hydra/config.yaml b/outputs/2026-07-10/11-07-57/.hydra/config.yaml deleted file mode 100644 index 024cbcfd2e5d40b89d32122ae2898bbb2bfdd2f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-07-57/.hydra/config.yaml +++ /dev/null @@ -1,218 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_noise: - enabled: true - mode: random_cleaner_fraction - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-10/11-07-57/.hydra/hydra.yaml b/outputs/2026-07-10/11-07-57/.hydra/hydra.yaml deleted file mode 100644 index a83f2de971ec639f20fe45736434e1d35ad1c200..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-07-57/.hydra/hydra.yaml +++ /dev/null @@ -1,212 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=random_cleaner_fraction - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_max_fraction=0.50 - - algorithm.memory_noise.revisit_max_fraction=0.50 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.validation.val_every_n_step=2000 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_max_fraction=0.50,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=random_cleaner_fraction,algorithm.memory_noise.revisit_max_fraction=0.50,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-10/11-07-57 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-10/11-07-57/.hydra/overrides.yaml b/outputs/2026-07-10/11-07-57/.hydra/overrides.yaml deleted file mode 100644 index b30d277b6349296325e79fe83f661bca8f3765a3..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-07-57/.hydra/overrides.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- +name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=random_cleaner_fraction -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_max_fraction=0.50 -- algorithm.memory_noise.revisit_max_fraction=0.50 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.validation.val_every_n_step=2000 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-10/11-07-57/main.log b/outputs/2026-07-10/11-07-57/main.log deleted file mode 100644 index 633d067f4db8ed71f3cea2f863d66f1d04d6fd02..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-07-57/main.log +++ /dev/null @@ -1,97 +0,0 @@ -[2026-07-10 11:08:59,514][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-10 11:08:59,689][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-10 11:08:59,690][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-10 11:08:59,690][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-10 11:08:59,690][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-10 11:08:59,690][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-10 11:10:51,417][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-10 11:10:51,427][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-10 11:10:51,433][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-10 11:10:51,465][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-10 11:10:51,487][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-10 11:10:51,506][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-10 11:10:51,550][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-10 11:10:51,740][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-10 11:10:57,124][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-10 11:11:02,391][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step112000.ckpt -[2026-07-10 11:11:25,397][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,400][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,402][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,402][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,407][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,411][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,413][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:25,455][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:11:26,386][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 1.0 B -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -437 M Trainable params -839 M Non-trainable params -1.3 B Total params -5,107.676 Total estimated model params size (MB) -[2026-07-10 11:11:27,399][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step112000.ckpt -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,775][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:11:30,774][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:22:02,333][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,930][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,930][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,930][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,930][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,930][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,931][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,932][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,933][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,933][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:17,933][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/generation_length_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 14:19:05,436][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=120000` reached. diff --git a/outputs/2026-07-10/11-09-21/.hydra/config.yaml b/outputs/2026-07-10/11-09-21/.hydra/config.yaml deleted file mode 100644 index da455977ca073cd59b672c4c743d3c545a8898f6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-21/.hydra/config.yaml +++ /dev/null @@ -1,225 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-10/11-09-21/.hydra/hydra.yaml b/outputs/2026-07-10/11-09-21/.hydra/hydra.yaml deleted file mode 100644 index f4c62c559e870de09228adbce5db6f8bda31248c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-21/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-10/11-09-21 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-10/11-09-21/.hydra/overrides.yaml b/outputs/2026-07-10/11-09-21/.hydra/overrides.yaml deleted file mode 100644 index f64ce23a93228172d6a76c751896c3e0f6141614..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-21/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-10/11-09-21/main.log b/outputs/2026-07-10/11-09-21/main.log deleted file mode 100644 index 4f6630a17d9f4e35fc7bca15c0638f78531832b4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-21/main.log +++ /dev/null @@ -1,3 +0,0 @@ -[2026-07-10 11:12:23,342][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-10 11:12:55,460][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-10/11-09-22/.hydra/config.yaml b/outputs/2026-07-10/11-09-22/.hydra/config.yaml deleted file mode 100644 index da455977ca073cd59b672c4c743d3c545a8898f6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-22/.hydra/config.yaml +++ /dev/null @@ -1,225 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 120000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 120000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - pose_similarity_threshold: 0.6 - training_use_plucker: true - training_plucker_weight: 1.0 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: fov_greedy - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - focal_length: 0.35 - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-10/11-09-22/.hydra/hydra.yaml b/outputs/2026-07-10/11-09-22/.hydra/hydra.yaml deleted file mode 100644 index 9274707308dbee6001cbe4592e990144e37b6e29..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-22/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=fov_greedy - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=120000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=fov_greedy,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]},experiment.training.max_steps=120000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-10/11-09-22 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-10/11-09-22/.hydra/overrides.yaml b/outputs/2026-07-10/11-09-22/.hydra/overrides.yaml deleted file mode 100644 index f64ce23a93228172d6a76c751896c3e0f6141614..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-22/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=fov_greedy -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=120000 diff --git a/outputs/2026-07-10/11-09-22/main.log b/outputs/2026-07-10/11-09-22/main.log deleted file mode 100644 index 5af77c1d8461cf44d6d55e713b6a6f863808c401..0000000000000000000000000000000000000000 --- a/outputs/2026-07-10/11-09-22/main.log +++ /dev/null @@ -1,84 +0,0 @@ -[2026-07-10 11:10:27,557][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-10 11:10:27,706][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-10 11:10:27,706][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-10 11:10:27,707][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-10 11:10:27,707][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-10 11:10:27,707][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-10 11:12:23,287][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-10 11:12:23,288][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-10 11:12:23,292][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-10 11:12:23,316][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-10 11:12:23,363][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-10 11:12:23,383][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-10 11:12:23,624][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-10 11:12:30,003][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-10 11:12:35,202][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/checkpoints/epoch2_step118000.ckpt -[2026-07-10 11:12:55,466][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,529][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,536][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,542][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,544][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,551][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:55,600][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-10 11:12:58,876][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-10 11:12:59,797][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/checkpoints/epoch2_step118000.ckpt -[2026-07-10 11:13:00,646][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:13:00,647][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-10 11:22:38,659][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,910][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,910][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,910][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,910][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,910][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,912][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,912][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 11:22:38,912][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-10 12:04:25,679][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=120000` reached. diff --git a/outputs/2026-07-14/20-11-44/.hydra/config.yaml b/outputs/2026-07-14/20-11-44/.hydra/config.yaml deleted file mode 100644 index 69764c8f248b29d03fca0ef1651010ccba69d4f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-44/.hydra/config.yaml +++ /dev/null @@ -1,228 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-14/20-11-44/.hydra/hydra.yaml b/outputs/2026-07-14/20-11-44/.hydra/hydra.yaml deleted file mode 100644 index 002ebaf408a077e8ba585bcdd0e33941153ae2bf..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-44/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-14/20-11-44 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-14/20-11-44/.hydra/overrides.yaml b/outputs/2026-07-14/20-11-44/.hydra/overrides.yaml deleted file mode 100644 index 65d50ff618ec1345f166c53bf26330c31a08ad9c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-44/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-14/20-11-44/main.log b/outputs/2026-07-14/20-11-44/main.log deleted file mode 100644 index 4112bc37fffd38256bfb2e4d87af02f3787c8a88..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-44/main.log +++ /dev/null @@ -1,22 +0,0 @@ -[2026-07-14 20:13:39,379][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-14 20:13:39,379][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:24:39,472][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:24:39,472][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:25:08,407][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:08,411][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:19,458][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:19,466][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:09,962][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:10,138][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-14/20-11-47/.hydra/config.yaml b/outputs/2026-07-14/20-11-47/.hydra/config.yaml deleted file mode 100644 index 69764c8f248b29d03fca0ef1651010ccba69d4f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-47/.hydra/config.yaml +++ /dev/null @@ -1,228 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-14/20-11-47/.hydra/hydra.yaml b/outputs/2026-07-14/20-11-47/.hydra/hydra.yaml deleted file mode 100644 index 2741a803347bc24d9bff3dcd48d5c901b5511eee..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-47/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-14/20-11-47 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-14/20-11-47/.hydra/overrides.yaml b/outputs/2026-07-14/20-11-47/.hydra/overrides.yaml deleted file mode 100644 index 65d50ff618ec1345f166c53bf26330c31a08ad9c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-47/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-14/20-11-47/main.log b/outputs/2026-07-14/20-11-47/main.log deleted file mode 100644 index 6a6570457aaab58efb542ef140621f9dd5228196..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-47/main.log +++ /dev/null @@ -1,11 +0,0 @@ -[2026-07-14 20:13:39,376][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:24:39,471][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:25:08,294][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:19,463][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:09,849][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-14/20-11-48/.hydra/config.yaml b/outputs/2026-07-14/20-11-48/.hydra/config.yaml deleted file mode 100644 index 69764c8f248b29d03fca0ef1651010ccba69d4f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-48/.hydra/config.yaml +++ /dev/null @@ -1,228 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-14/20-11-48/.hydra/hydra.yaml b/outputs/2026-07-14/20-11-48/.hydra/hydra.yaml deleted file mode 100644 index 2e5fbfeab1fac15e2dfd15166303ab01e002da43..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-48/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-14/20-11-48 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-14/20-11-48/.hydra/overrides.yaml b/outputs/2026-07-14/20-11-48/.hydra/overrides.yaml deleted file mode 100644 index 65d50ff618ec1345f166c53bf26330c31a08ad9c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-48/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-14/20-11-48/main.log b/outputs/2026-07-14/20-11-48/main.log deleted file mode 100644 index 2aa3c9f5869f1a2573768375a9c1996cd1fbc604..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-48/main.log +++ /dev/null @@ -1,155 +0,0 @@ -[2026-07-14 20:12:54,296][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-14 20:12:54,296][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, blocks.0.r_mlp.fc1.weight, blocks.0.r_mlp.fc1.bias, ... (+300 more) -[2026-07-14 20:12:54,386][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-14 20:12:54,387][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-14 20:12:54,472][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-14 20:12:54,510][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-14 20:12:54,567][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-14 20:12:54,772][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-14 20:12:54,773][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-14 20:12:54,773][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-14 20:12:54,773][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-14 20:12:54,773][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-14 20:13:39,377][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-14 20:13:39,381][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-14 20:13:39,381][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-14 20:13:40,063][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-14 20:13:47,346][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:56,509][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 20:13:56,510][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 20:51:57,962][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,596][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,597][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,598][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 20:52:11,598][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-14 23:24:27,800][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=10000` reached. -[2026-07-14 23:24:39,471][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:24:39,471][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:24:39,472][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:24:39,472][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-14 23:24:39,472][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:24:39,909][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-14 23:24:39,909][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-14 23:24:39,909][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-14 23:24:39,909][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-14 23:24:39,910][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-14 23:25:08,177][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/checkpoints exists and is not empty. - -[2026-07-14 23:25:08,285][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-14 23:25:08,294][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:08,295][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:08,401][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:08,409][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:19,462][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:19,463][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:19,464][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:19,464][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:20,166][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-14 23:25:20,501][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:25:21,003][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-15 10:05:27,767][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-15 10:05:51,297][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-15 10:05:53,388][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-15 10:05:53,388][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-15 10:05:53,388][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-15 10:05:53,388][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-15 10:05:53,388][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-15 10:07:49,987][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-15 10:08:09,946][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:10,000][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:10,073][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:10,163][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:10,891][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.297 Total estimated model params size (MB) -[2026-07-15 10:08:11,917][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:46:56,515][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=200000` reached. diff --git a/outputs/2026-07-14/20-11-49/.hydra/config.yaml b/outputs/2026-07-14/20-11-49/.hydra/config.yaml deleted file mode 100644 index 69764c8f248b29d03fca0ef1651010ccba69d4f4..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-49/.hydra/config.yaml +++ /dev/null @@ -1,228 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: 500 - memory_noise: - enabled: true - mode: source_conditioned - anchor_max_fraction: 0.25 - dynamic_clean_max_fraction: 0.25 - revisit_clean_max_fraction: 0.25 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_source_embeddings: true - memory_attention_key_only_geometry: true - add_timestamp_embedding: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -name: train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-14/20-11-49/.hydra/hydra.yaml b/outputs/2026-07-14/20-11-49/.hydra/hydra.yaml deleted file mode 100644 index 43d7f89278c0dc73ff8afab1d7aa9530b8c33cc8..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-49/.hydra/hydra.yaml +++ /dev/null @@ -1,223 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_source_embeddings=true - - algorithm.memory_attention_key_only_geometry=true - - algorithm.add_timestamp_embedding=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_conditioned - - algorithm.memory_noise.anchor_max_fraction=0.25 - - algorithm.memory_noise.dynamic_clean_max_fraction=0.25 - - algorithm.memory_noise.revisit_clean_max_fraction=0.25 - - algorithm.memory_noise.dynamic_generated_max_fraction=0.5 - - algorithm.memory_noise.revisit_generated_max_fraction=0.5 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.add_timestamp_embedding=false,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.anchor_max_fraction=0.25,algorithm.memory_noise.dynamic_clean_max_fraction=0.25,algorithm.memory_noise.dynamic_generated_max_fraction=0.5,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_conditioned,algorithm.memory_noise.revisit_clean_max_fraction=0.25,algorithm.memory_noise.revisit_generated_max_fraction=0.5,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_source_embeddings=true,algorithm.memory_source_simulation.training_generated_suffix_max_frames=500,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-14/20-11-49 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-14/20-11-49/.hydra/overrides.yaml b/outputs/2026-07-14/20-11-49/.hydra/overrides.yaml deleted file mode 100644 index 65d50ff618ec1345f166c53bf26330c31a08ad9c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-49/.hydra/overrides.yaml +++ /dev/null @@ -1,65 +0,0 @@ -- +name=train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/ -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_source_embeddings=true -- algorithm.memory_attention_key_only_geometry=true -- algorithm.add_timestamp_embedding=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=500 -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_conditioned -- algorithm.memory_noise.anchor_max_fraction=0.25 -- algorithm.memory_noise.dynamic_clean_max_fraction=0.25 -- algorithm.memory_noise.revisit_clean_max_fraction=0.25 -- algorithm.memory_noise.dynamic_generated_max_fraction=0.5 -- algorithm.memory_noise.revisit_generated_max_fraction=0.5 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-14/20-11-49/main.log b/outputs/2026-07-14/20-11-49/main.log deleted file mode 100644 index 5cefc86d7b239f5e50e5dd724d204bb86e1deec6..0000000000000000000000000000000000000000 --- a/outputs/2026-07-14/20-11-49/main.log +++ /dev/null @@ -1,11 +0,0 @@ -[2026-07-14 20:13:39,375][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-14 20:13:52,879][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 20:13:56,511][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-14 23:24:39,471][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-14 23:25:08,406][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-14 23:25:19,459][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-14 23:25:20,505][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 10:08:09,851][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 10:08:12,611][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-15/00-45-53/.hydra/config.yaml b/outputs/2026-07-15/00-45-53/.hydra/config.yaml deleted file mode 100644 index ce242abe530a0a180db7f1628c3b60cffcd95b66..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-53/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: false - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-15/00-45-53/.hydra/hydra.yaml b/outputs/2026-07-15/00-45-53/.hydra/hydra.yaml deleted file mode 100644 index 251139a95ce9a5c69b806236943fde38f8bb4f0a..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-53/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=false - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=false,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-15/00-45-53 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-15/00-45-53/.hydra/overrides.yaml b/outputs/2026-07-15/00-45-53/.hydra/overrides.yaml deleted file mode 100644 index 4d16bb2f69fa3890671e56a11c4fb41f680f0282..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-53/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=false -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-15/00-45-53/main.log b/outputs/2026-07-15/00-45-53/main.log deleted file mode 100644 index fa0b384860a48ca6d35597f477fd70cf344368c1..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-53/main.log +++ /dev/null @@ -1,186 +0,0 @@ -[2026-07-15 00:46:27,937][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-15 00:46:28,076][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-15 00:46:28,143][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-15 00:46:28,470][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-15 00:46:53,787][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-15 00:46:53,788][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, memory_age_embed.weight, blocks.0.r_mlp.fc1.weight, ... (+301 more) -[2026-07-15 00:46:53,872][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-15 00:46:53,873][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-15 00:46:53,941][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-15 00:46:53,973][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-15 00:46:54,037][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-15 00:46:54,171][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-15 00:46:54,171][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-15 00:46:54,171][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-15 00:46:54,171][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-15 00:46:54,172][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-15 00:47:29,266][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-15 00:47:29,272][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-15 00:47:29,274][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-15 00:47:29,274][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-15 00:47:29,293][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-15 00:47:29,294][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-15 00:47:29,298][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-15 00:47:30,328][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-15 00:47:35,081][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:42,360][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 01:26:54,374][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,478][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,478][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,478][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,478][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,478][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,479][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,480][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,480][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,480][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 01:27:06,480][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-15 04:04:06,814][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=10000` reached. -[2026-07-15 04:04:20,620][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:22,056][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-15 04:04:22,057][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-15 04:04:22,057][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-15 04:04:22,057][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-15 04:04:22,057][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-15 04:04:46,824][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-15 04:04:46,900][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-15 04:04:46,987][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-15 04:04:47,299][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-15 04:04:47,414][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/checkpoints exists and is not empty. - -[2026-07-15 04:04:47,414][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-15 04:04:47,422][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,422][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,426][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,430][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:05:00,328][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,328][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,328][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,331][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:01,053][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-15 04:05:01,390][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,844][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-15 15:04:22,147][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-15 15:04:43,999][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-15 15:04:46,496][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-15 15:04:46,497][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-15 15:04:46,497][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-15 15:04:46,497][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-15 15:04:46,497][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-15 15:06:32,098][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-15 15:06:32,225][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-15 15:06:32,232][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-15 15:06:32,599][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-15 15:06:32,608][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-15 15:06:53,799][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,155][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,196][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,291][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,998][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-15 15:06:55,966][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,991][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-07-18 00:46:05,088][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-07-18 00:46:05,089][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-07-15/00-45-54/.hydra/config.yaml b/outputs/2026-07-15/00-45-54/.hydra/config.yaml deleted file mode 100644 index ce242abe530a0a180db7f1628c3b60cffcd95b66..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-54/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: false - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-15/00-45-54/.hydra/hydra.yaml b/outputs/2026-07-15/00-45-54/.hydra/hydra.yaml deleted file mode 100644 index a565637b259997ca68e6bb7820ddb3d205698ecf..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-54/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=false - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=false,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-15/00-45-54 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-15/00-45-54/.hydra/overrides.yaml b/outputs/2026-07-15/00-45-54/.hydra/overrides.yaml deleted file mode 100644 index 4d16bb2f69fa3890671e56a11c4fb41f680f0282..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-54/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=false -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-15/00-45-54/main.log b/outputs/2026-07-15/00-45-54/main.log deleted file mode 100644 index 92c97390815c6e7ed21932fc052576d95ecdc786..0000000000000000000000000000000000000000 --- a/outputs/2026-07-15/00-45-54/main.log +++ /dev/null @@ -1,76 +0,0 @@ -[2026-07-15 00:46:27,938][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-15 00:46:28,240][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-15 00:46:28,416][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-15 00:46:28,804][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-15 00:47:29,264][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-15 00:47:29,265][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-15 00:47:29,266][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-15 00:47:29,465][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-15 00:47:29,475][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-15 00:47:30,332][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-15 00:47:31,081][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-15 00:47:31,327][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:40,421][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 00:47:43,039][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:04:20,620][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:20,622][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-15 04:04:46,836][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-15 04:04:46,870][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-15 04:04:46,905][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-15 04:04:46,970][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-15 04:04:47,424][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,428][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,432][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:04:47,434][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-15 04:05:00,320][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,322][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,325][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:00,329][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 04:05:01,396][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:32,111][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-15 15:06:32,153][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-15 15:06:32,379][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-15 15:06:32,566][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-15 15:06:54,020][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,165][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,211][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:54,236][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-15 15:06:55,990][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,086][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-07-18 00:46:05,087][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-07-18 00:46:05,090][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-18 00:46:05,104][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-07-18 00:46:05,104][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-07-17/18-47-36/.hydra/config.yaml b/outputs/2026-07-17/18-47-36/.hydra/config.yaml deleted file mode 100644 index 84c6a1f71a3a62f073b8f7e0bac7385cb564e117..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-36/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-17/18-47-36/.hydra/hydra.yaml b/outputs/2026-07-17/18-47-36/.hydra/hydra.yaml deleted file mode 100644 index a74a939d034f465a320cf31142d5d0c44b20d11d..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-36/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=false,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-17/18-47-36 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-17/18-47-36/.hydra/overrides.yaml b/outputs/2026-07-17/18-47-36/.hydra/overrides.yaml deleted file mode 100644 index 5cad5dbdebf3980c008fde4c3141f3091d8d312c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-36/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-17/18-47-36/main.log b/outputs/2026-07-17/18-47-36/main.log deleted file mode 100644 index 1e93952aa6fd0ebec728e23240169115b9b76050..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-36/main.log +++ /dev/null @@ -1,45 +0,0 @@ -[2026-07-17 18:48:04,501][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 18:48:04,724][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 18:48:05,197][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 18:48:44,269][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 18:48:44,271][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 18:48:44,299][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 18:48:44,311][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-17 18:48:44,749][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-17 18:48:45,625][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:14,599][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:43,027][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 22:03:43,136][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 22:03:43,246][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 22:03:43,373][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,377][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:46,430][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,436][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,782][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:31,666][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 08:55:31,765][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 08:55:32,203][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 08:55:49,607][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,672][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,778][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,469][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. diff --git a/outputs/2026-07-17/18-47-37/.hydra/config.yaml b/outputs/2026-07-17/18-47-37/.hydra/config.yaml deleted file mode 100644 index 84c6a1f71a3a62f073b8f7e0bac7385cb564e117..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-37/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: false - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-17/18-47-37/.hydra/hydra.yaml b/outputs/2026-07-17/18-47-37/.hydra/hydra.yaml deleted file mode 100644 index 35839a28e30c70030bb89e21761c7bb563ada069..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-37/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=false - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=false,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-17/18-47-37 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-17/18-47-37/.hydra/overrides.yaml b/outputs/2026-07-17/18-47-37/.hydra/overrides.yaml deleted file mode 100644 index 5cad5dbdebf3980c008fde4c3141f3091d8d312c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-37/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=false -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-17/18-47-37/main.log b/outputs/2026-07-17/18-47-37/main.log deleted file mode 100644 index 699289a144171934efc33ac2e2f3da94ba935239..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/18-47-37/main.log +++ /dev/null @@ -1,186 +0,0 @@ -[2026-07-17 18:48:04,461][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 18:48:04,580][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 18:48:04,746][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 18:48:04,874][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 18:48:04,914][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 18:48:29,146][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-17 18:48:29,147][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, memory_age_embed.weight, blocks.0.r_mlp.fc1.weight, ... (+301 more) -[2026-07-17 18:48:29,234][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-17 18:48:29,235][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-17 18:48:29,327][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-17 18:48:29,364][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-17 18:48:29,420][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-17 18:48:29,561][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-17 18:48:29,561][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-17 18:48:29,561][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-17 18:48:29,561][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-17 18:48:29,562][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-17 18:48:44,269][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 18:48:44,278][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 18:48:44,294][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 18:48:44,296][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-17 18:48:44,305][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-17 18:48:44,898][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 18:48:44,913][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-17 18:48:44,921][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-17 18:48:57,501][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 18:48:57,522][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-17 18:49:03,203][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:08,784][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 18:49:10,463][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 18:49:11,454][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 19:27:46,772][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,444][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,444][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,444][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,445][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 19:27:59,446][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 22:03:02,002][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=10000` reached. -[2026-07-17 22:03:14,599][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,600][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 22:03:14,911][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-17 22:03:14,912][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-17 22:03:14,912][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-17 22:03:14,912][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-17 22:03:14,912][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-17 22:03:43,082][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 22:03:43,092][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 22:03:43,115][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 22:03:43,119][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 22:03:43,208][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 22:03:43,363][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/checkpoints exists and is not empty. - -[2026-07-17 22:03:43,363][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-17 22:03:43,375][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,379][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,381][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,384][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:43,385][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 22:03:46,431][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,432][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,435][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,438][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:46,593][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 22:03:47,465][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-17 22:03:47,799][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:47,808][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 22:03:48,308][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-18 08:53:54,423][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-18 08:54:16,527][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-18 08:54:17,349][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-18 08:54:17,349][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-18 08:54:17,349][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-18 08:54:17,349][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-18 08:54:17,349][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-18 08:55:31,648][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 08:55:31,676][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 08:55:31,774][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 08:55:31,779][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 08:55:32,070][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 08:55:32,212][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 08:55:49,006][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,466][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,482][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,628][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:49,717][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 08:55:50,470][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-18 08:55:51,448][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_countcorr_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 08:55:51,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-20 18:20:12,133][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=200000` reached. diff --git a/outputs/2026-07-17/20-20-55/.hydra/config.yaml b/outputs/2026-07-17/20-20-55/.hydra/config.yaml deleted file mode 100644 index 1a7fb5307bee9d6a98cf01c653319c9750f51439..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-55/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: false - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-17/20-20-55/.hydra/hydra.yaml b/outputs/2026-07-17/20-20-55/.hydra/hydra.yaml deleted file mode 100644 index c26123117e9f3ce494420437601abbcb8b5d8cc8..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-55/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-17/20-20-55 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-17/20-20-55/.hydra/overrides.yaml b/outputs/2026-07-17/20-20-55/.hydra/overrides.yaml deleted file mode 100644 index 132373229e24f12a173c6dd4448ca44267af9b52..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-55/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-17/20-20-55/main.log b/outputs/2026-07-17/20-20-55/main.log deleted file mode 100644 index ccd4191fafc7f8cb540fdc902ec2125c09cf13c3..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-55/main.log +++ /dev/null @@ -1,212 +0,0 @@ -[2026-07-17 20:21:33,079][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 20:21:33,079][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 20:21:33,111][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 20:21:33,332][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 20:21:33,386][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 20:21:33,594][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 20:21:59,990][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-17 20:21:59,991][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, memory_age_embed.weight, blocks.0.r_mlp.fc1.weight, ... (+301 more) -[2026-07-17 20:22:00,079][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-17 20:22:00,080][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-17 20:22:00,125][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-17 20:22:00,155][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-17 20:22:00,195][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-17 20:22:00,341][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-17 20:22:00,341][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-17 20:22:00,341][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-17 20:22:00,341][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-17 20:22:00,342][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-17 20:22:36,732][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 20:22:36,733][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 20:22:36,733][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 20:22:36,736][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 20:22:36,742][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 20:22:36,762][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-17 20:22:36,763][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-17 20:22:37,126][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-17 20:22:37,575][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-17 20:22:37,581][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 20:22:37,595][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-17 20:22:37,619][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-17 20:22:43,016][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:49,570][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,926][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 21:02:06,335][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,113][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,113][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,113][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,113][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,114][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,115][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,115][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,115][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,115][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 21:02:12,115][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-17 23:39:10,050][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=10000` reached. -[2026-07-17 23:39:20,875][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-17 23:39:20,875][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,875][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,875][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,876][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,876][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,876][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:21,142][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-17 23:39:21,142][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-17 23:39:21,143][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-17 23:39:21,143][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-17 23:39:21,143][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-17 23:39:49,503][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-17 23:39:49,523][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-17 23:39:49,555][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-17 23:39:49,571][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-17 23:39:49,661][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-17 23:39:50,265][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-17 23:39:50,381][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/checkpoints exists and is not empty. - -[2026-07-17 23:39:50,382][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-17 23:39:50,386][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,396][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,399][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,401][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,403][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,403][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:40:02,153][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,155][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,158][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,158][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,159][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,164][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,848][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-17 23:40:03,205][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,628][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-18 10:36:55,971][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-18 10:37:16,851][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-18 10:37:19,186][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-18 10:37:19,186][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-18 10:37:19,186][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-18 10:37:19,186][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-18 10:37:19,186][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-18 10:39:05,856][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 10:39:05,873][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 10:39:05,915][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 10:39:05,967][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 10:39:06,036][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 10:39:06,184][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 10:39:06,281][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 10:39:22,483][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:22,751][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:22,788][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:22,865][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:22,965][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:23,036][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:23,734][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-18 10:39:24,697][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 10:39:24,790][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,790][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,790][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,791][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,791][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,791][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-07-20 20:20:56,442][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-07-20 20:20:56,442][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-07-20 20:20:56,443][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,443][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-07-20 20:20:56,444][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,444][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-07-17/20-20-56/.hydra/config.yaml b/outputs/2026-07-17/20-20-56/.hydra/config.yaml deleted file mode 100644 index 1a7fb5307bee9d6a98cf01c653319c9750f51439..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-56/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: false - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-17/20-20-56/.hydra/hydra.yaml b/outputs/2026-07-17/20-20-56/.hydra/hydra.yaml deleted file mode 100644 index f50827871b721fbdcc1f74c6abfa6883a7719f3d..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-56/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=false - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=false,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-17/20-20-56 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-17/20-20-56/.hydra/overrides.yaml b/outputs/2026-07-17/20-20-56/.hydra/overrides.yaml deleted file mode 100644 index 132373229e24f12a173c6dd4448ca44267af9b52..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-56/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=false -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-17/20-20-56/main.log b/outputs/2026-07-17/20-20-56/main.log deleted file mode 100644 index 61e02592d9245acbe19c4bbfeb414af4a5b265a2..0000000000000000000000000000000000000000 --- a/outputs/2026-07-17/20-20-56/main.log +++ /dev/null @@ -1,34 +0,0 @@ -[2026-07-17 20:21:33,313][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 20:21:33,466][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 20:22:36,732][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 20:22:36,841][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 20:22:36,853][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-17 20:22:38,075][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:47,961][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 20:22:50,925][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:39:20,875][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:20,876][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-17 23:39:49,532][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-17 23:39:49,567][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-17 23:39:50,392][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:39:50,394][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-17 23:40:02,159][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:02,159][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-17 23:40:03,208][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:05,943][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 10:39:06,033][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 10:39:22,638][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:23,010][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 10:39:24,790][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 10:39:24,791][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-07-20 20:20:56,441][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-07-20 20:20:56,443][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-20 20:20:56,443][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-07-18/00-46-53/.hydra/config.yaml b/outputs/2026-07-18/00-46-53/.hydra/config.yaml deleted file mode 100644 index 75ad480d630dbab8223b46ed26a1a096ea62d1cf..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-53/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-18/00-46-53/.hydra/hydra.yaml b/outputs/2026-07-18/00-46-53/.hydra/hydra.yaml deleted file mode 100644 index 3dff69be154cee587303713e7464dd6778bbe4d7..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-53/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-18/00-46-53 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-18/00-46-53/.hydra/overrides.yaml b/outputs/2026-07-18/00-46-53/.hydra/overrides.yaml deleted file mode 100644 index 56018cf8195f9be151815a3c8e6f6e4e8fa0508c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-53/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-18/00-46-53/main.log b/outputs/2026-07-18/00-46-53/main.log deleted file mode 100644 index cdaea20c6c196f3648ef8cc3630c1647451ed278..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-53/main.log +++ /dev/null @@ -1,195 +0,0 @@ -[2026-07-18 00:47:16,254][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 00:47:16,295][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 00:47:16,457][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 00:47:16,491][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 00:47:16,972][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 00:47:39,241][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-18 00:47:39,242][pytorch_lightning.utilities.rank_zero][INFO] - Missing checkpoint keys: spatial_rotary_emb.freqs, temporal_rotary_emb.freqs, pose_embedder.model.0.weight, pose_embedder.model.0.bias, pose_embedder.model.2.weight, pose_embedder.model.2.bias, memory_age_embed.weight, blocks.0.r_mlp.fc1.weight, ... (+301 more) -[2026-07-18 00:47:39,325][pytorch_lightning.utilities.rank_zero][INFO] - Zeroed frame-memory gate slices for base checkpoint load: blocks.0.r_adaLN_modulation.1.weight, blocks.0.r_adaLN_modulation.1.bias, blocks.1.r_adaLN_modulation.1.weight, blocks.1.r_adaLN_modulation.1.bias, blocks.2.r_adaLN_modulation.1.weight, blocks.2.r_adaLN_modulation.1.bias, blocks.3.r_adaLN_modulation.1.weight, blocks.3.r_adaLN_modulation.1.bias, ... (+24 more) -[2026-07-18 00:47:39,325][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors: 302 tensors. -[2026-07-18 00:47:39,396][pytorch_lightning.utilities.rank_zero][INFO] - Checkpoint prefix match: strip '', add ''. -[2026-07-18 00:47:39,430][pytorch_lightning.utilities.rank_zero][INFO] - Model weights loaded from /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors: 228 tensors. -[2026-07-18 00:47:39,480][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-18 00:47:39,609][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-18 00:47:39,610][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-18 00:47:39,610][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-18 00:47:39,610][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-18 00:47:39,610][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-18 00:47:53,255][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 00:47:53,255][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 00:47:53,255][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 00:47:53,257][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 00:47:53,277][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/8 -[2026-07-18 00:47:54,186][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 2, MEMBER: 3/8 -[2026-07-18 00:47:54,393][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 7, MEMBER: 8/8 -[2026-07-18 00:47:55,206][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 5, MEMBER: 6/8 -[2026-07-18 00:48:04,621][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 00:48:04,639][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/8 -[2026-07-18 00:48:10,725][pytorch_lightning.utilities.rank_zero][INFO] - ---------------------------------------------------------------------------------------------------- -distributed_backend=nccl -All distributed processes registered. Starting with 8 processes ----------------------------------------------------------------------------------------------------- - -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,107][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,107][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:18,583][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -302 M Trainable params -839 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-18 00:48:19,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,589][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,590][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,590][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 01:27:32,956][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('validation/latent_mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,171][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0000_0100', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,172][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0100_0200', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0200_0300', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0300_0400', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/mse_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,173][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/psnr_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 01:27:38,174][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:433: It is recommended to use `self.log('segment/lpips_0400_0500', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices. - -[2026-07-18 04:04:28,192][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=10000` reached. -[2026-07-18 04:04:40,544][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-18 04:04:40,544][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:43,168][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-18 04:04:43,168][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-18 04:04:43,168][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-18 04:04:43,168][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-18 04:04:43,169][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-18 04:05:08,312][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 04:05:08,315][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 04:05:08,329][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 04:05:08,494][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 04:05:08,786][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 04:05:08,900][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:639: Checkpoint directory /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/checkpoints exists and is not empty. - -[2026-07-18 04:05:08,901][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-18 04:05:08,909][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,918][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,926][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,927][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,928][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:20,355][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,357][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,357][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,362][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,367][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:21,129][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-18 04:05:21,457][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_01_no_updown_memory_only_step10000.ckpt -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,945][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:154: You're resuming from a checkpoint that ended before the epoch ended. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint - -[2026-07-18 15:03:52,032][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer.fit` stopped: `max_steps=40000` reached. -[2026-07-18 15:04:12,865][pytorch_lightning.utilities.rank_zero][INFO] - Using 16bit Automatic Mixed Precision (AMP) -[2026-07-18 15:04:13,105][pytorch_lightning.utilities.rank_zero][INFO] - GPU available: True (cuda), used: True -[2026-07-18 15:04:13,105][pytorch_lightning.utilities.rank_zero][INFO] - TPU available: False, using: 0 TPU cores -[2026-07-18 15:04:13,105][pytorch_lightning.utilities.rank_zero][INFO] - IPU available: False, using: 0 IPUs -[2026-07-18 15:04:13,106][pytorch_lightning.utilities.rank_zero][INFO] - HPU available: False, using: 0 HPUs -[2026-07-18 15:04:13,106][pytorch_lightning.utilities.rank_zero][INFO] - `Trainer(limit_val_batches=1)` was configured so 1 batch will be used. -[2026-07-18 15:05:23,418][lightning.fabric.utilities.seed][INFO] - [rank: 1] Seed set to 42 -[2026-07-18 15:05:23,524][lightning.fabric.utilities.seed][INFO] - [rank: 7] Seed set to 42 -[2026-07-18 15:05:23,875][lightning.fabric.utilities.seed][INFO] - [rank: 2] Seed set to 42 -[2026-07-18 15:05:24,240][lightning.fabric.utilities.seed][INFO] - [rank: 5] Seed set to 42 -[2026-07-18 15:05:24,345][lightning.fabric.utilities.seed][INFO] - [rank: 0] Seed set to 42 -[2026-07-18 15:05:24,353][pytorch_lightning.utilities.rank_zero][INFO] - Restoring states from the checkpoint path at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 15:05:43,351][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 2 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,411][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,537][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,564][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 5 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,605][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 7 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:44,310][lightning.pytorch.callbacks.model_summary][INFO] - - | Name | Type | Params ---------------------------------------------------------------------------------- -0 | diffusion_model | Diffusion | 910 M -1 | validation_lpips_model | LearnedPerceptualImagePatchSimilarity | 2.5 M -2 | vae | AutoencoderKL | 229 M ---------------------------------------------------------------------------------- -910 M Trainable params -231 M Non-trainable params -1.1 B Total params -4,569.428 Total estimated model params size (MB) -[2026-07-18 15:05:45,287][pytorch_lightning.utilities.rank_zero][INFO] - Restored all states from the checkpoint at /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/checkpoints/curriculum_02_no_updown_step40000.ckpt -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,311][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 1] Received SIGTERM: 15 -[2026-07-21 00:47:00,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 5] Received SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 2] Received SIGTERM: 15 -[2026-07-21 00:47:00,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,367][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 0] Received SIGTERM: 15 -[2026-07-21 00:47:00,368][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 7] Received SIGTERM: 15 -[2026-07-21 00:47:00,368][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/outputs/2026-07-18/00-46-54/.hydra/config.yaml b/outputs/2026-07-18/00-46-54/.hydra/config.yaml deleted file mode 100644 index 75ad480d630dbab8223b46ed26a1a096ea62d1cf..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-54/.hydra/config.yaml +++ /dev/null @@ -1,238 +0,0 @@ -experiment: - debug: ${debug} - tasks: - - training - num_nodes: 1 - training: - precision: 16-mixed - compile: false - lr: 2.0e-05 - batch_size: 16 - max_epochs: -1 - max_steps: 200000 - max_time: null - data: - num_workers: 4 - shuffle: true - optim: - accumulate_grad_batches: 1 - gradient_clip_val: 1.0 - checkpointing: - every_n_train_steps: 2000 - every_n_epochs: null - train_time_interval: null - save_last_k: 5 - enable_version_counter: false - curriculum: - enabled: true - stages: - - name: no_updown - until_step: 40000 - dataset: - wo_updown: true - - name: with_updown - until_step: 200000 - dataset: - wo_updown: false - validation: - precision: 16-mixed - compile: false - batch_size: 1 - val_every_n_step: 2000 - val_every_n_epoch: null - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - test: - precision: 16-mixed - compile: false - batch_size: 1 - limit_batch: 1 - inference_mode: false - data: - num_workers: 4 - shuffle: false - logging: - metrics: null -dataset: - debug: ${debug} - save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - metadata: null - n_frames: 8 - n_frames_valid: 1100 - frame_skip: 1 - resolution: 128 - image_hw: - - 360 - - 640 - vertical_fov_degrees: 70.0 - observation_shape: - - 16 - - 18 - - 32 - data_mean: 0.0 - data_std: 1.0 - action_cond_dim: 25 - context_length: 100 - single_eval_clip: true - shuffle_clips: true - memory_selection: - enabled: true - image_hw: ${..image_hw} - vertical_fov_degrees: ${..vertical_fov_degrees} - causal: true - max_anchor_frames: 2 - max_dynamic_frames: 4 - max_revisit_frames: 2 - training_dropout: 0.1 - fov_overlap_threshold: 0.6 - min_total_selected_coverage: 0.1 - local_context_exclusion_frames: 8 - plucker_moment_radius: 30.0 - anchor_diverse_selection: true - pose_preselect_topk: 32 - candidate_chunk_size: 0 - dynamic: - selection_policy: multiview - multiview_selector: shared_area_diverse - shared_area_threshold: 0.3 - min_pose_novelty: 0.1 - shortlist_per_reference: 16 - scene_threshold: 2.5 - state_threshold: 2.5 - stable_threshold: 1.0 - stable_frames: 3 - min_event_gap: 8 - min_anchor_score: 2.0 - max_event_anchors: null - b_pose: 0.3 - b_action: 0.2 - _name: video_minecraft_dememwm_latent -algorithm: - debug: ${debug} - lr: ${experiment.training.lr} - x_shape: ${dataset.observation_shape} - frame_stack: 1 - frame_skip: ${dataset.frame_skip} - data_mean: ${dataset.data_mean} - data_std: ${dataset.data_std} - external_cond_dim: 0 - context_frames: 600 - weight_decay: 0.002 - warmup_steps: 500 - optimizer_beta: - - 0.9 - - 0.99 - uncertainty_scale: 1 - guidance_scale: 0.0 - chunk_size: 1 - scheduling_matrix: autoregressive - noise_level: random_all - causal: true - diffusion: - objective: pred_v - beta_schedule: sigmoid - schedule_fn_kwargs: {} - clip_noise: 20.0 - use_snr: false - use_cum_snr: false - use_fused_snr: true - snr_clip: 5.0 - cum_snr_decay: 0.96 - timesteps: 1000 - sampling_timesteps: 20 - ddim_sampling_eta: 0.0 - stabilization_level: 15 - architecture: - network_size: 64 - attn_heads: 4 - attn_dim_head: 64 - dim_mults: - - 1 - - 2 - - 4 - - 8 - resolution: ${dataset.resolution} - attn_resolutions: - - 16 - - 32 - - 64 - - 128 - use_init_temporal_attn: true - use_linear_attn: true - time_emb_type: rotary - n_frames: ${dataset.n_frames} - metadata: ${dataset.metadata} - memory_selection: ${dataset.memory_selection} - memory_source_simulation: - training_generated_suffix_max_frames: null - memory_noise: - enabled: true - mode: source_free_logsnr - stable_probability: 0.25 - noisy_distribution: query_to_stabilization_uniform_logsnr_v1 - dynamic_generated_max_fraction: 0.5 - revisit_generated_max_fraction: 0.5 - anchor_max_fraction: 0.25 - dynamic_max_fraction: 0.5 - revisit_max_fraction: 0.25 - validation_noisy_memory: false - noise_route: - anchor: high - dynamic: low - revisit: all - trainability: - freeze_vae: true - reference_attention: true - adaln_mlp: true - geometry_projections: true - train_full_dit: true - full_dit_start_step: 10000 - lr: - memory_modules: 8.0e-05 - base_dit: 2.0e-05 - n_tokens: 8 - action_cond_dim: ${dataset.action_cond_dim} - pose_cond_dim: 5 - use_plucker: true - relative_embedding: true - state_embed_only_on_qk: true - use_memory_attention: true - unified_memory_attention: true - anchor_zero_geometry: false - memory_role_embedding: true - memory_source_embedding: false - memory_attention_key_only_geometry: true - memory_age: - enabled: true - num_buckets: 32 - num_exact_buckets: 17 - overflow_start: 100 - memory_stream_count_correction: true - ref_mode: sequential - vertical_fov_degrees: ${dataset.vertical_fov_degrees} - log_video: true - save_local: false - metric_report_segment: 100 - require_pose_prediction: false - _name: dememwm_base -debug: false -wandb: - entity: turlin - project: worldmem - mode: offline -resume: null -load: null -auto_resume: true -resume_ckpt_path: null -allow_dememwm_legacy_migration: false -name: train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k -output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ -seed: 42 -customized_load: true -seperate_load: true -diffusion_model_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -vae_path: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors diff --git a/outputs/2026-07-18/00-46-54/.hydra/hydra.yaml b/outputs/2026-07-18/00-46-54/.hydra/hydra.yaml deleted file mode 100644 index 1cb88eb5b24513b380b6921379532858e109e2c5..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-54/.hydra/hydra.yaml +++ /dev/null @@ -1,227 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - +name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k - - +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ - - +seed=42 - - auto_resume=true - - wandb.mode=offline - - experiment.tasks=[training] - - experiment.num_nodes=1 - - algorithm=dememwm_base - - +customized_load=true - - +seperate_load=true - - +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors - - +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors - - dataset=video_minecraft_dememwm_latent - - dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft - - dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features - - dataset.n_frames=8 - - dataset.n_frames_valid=1100 - - dataset.context_length=100 - - dataset.single_eval_clip=true - - dataset.memory_selection.enabled=true - - dataset.memory_selection.causal=true - - dataset.memory_selection.max_anchor_frames=2 - - dataset.memory_selection.max_dynamic_frames=4 - - dataset.memory_selection.max_revisit_frames=2 - - dataset.memory_selection.training_dropout=0.1 - - dataset.memory_selection.dynamic.selection_policy=multiview - - dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse - - algorithm.causal=true - - algorithm.n_tokens=8 - - algorithm.context_frames=600 - - algorithm.metric_report_segment=100 - - algorithm.use_memory_attention=true - - algorithm.anchor_zero_geometry=false - - algorithm.memory_role_embedding=true - - algorithm.memory_source_embedding=false - - algorithm.memory_attention_key_only_geometry=true - - algorithm.memory_age.enabled=true - - algorithm.memory_age.num_buckets=32 - - algorithm.memory_age.num_exact_buckets=17 - - algorithm.memory_age.overflow_start=100 - - algorithm.memory_stream_count_correction=true - - algorithm.noise_route.anchor=high - - algorithm.noise_route.dynamic=low - - algorithm.noise_route.revisit=all - - algorithm.memory_source_simulation.training_generated_suffix_max_frames=null - - algorithm.memory_noise.enabled=true - - algorithm.memory_noise.mode=source_free_logsnr - - algorithm.memory_noise.stable_probability=0.25 - - algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 - - algorithm.memory_noise.validation_noisy_memory=false - - algorithm.chunk_size=1 - - algorithm.warmup_steps=500 - - algorithm.trainability.train_full_dit=true - - algorithm.trainability.full_dit_start_step=10000 - - algorithm.trainability.lr.memory_modules=8.0e-5 - - algorithm.trainability.lr.base_dit=2.0e-5 - - algorithm.log_video=true - - algorithm.save_local=false - - algorithm.diffusion.sampling_timesteps=20 - - experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} - - experiment.training.batch_size=16 - - experiment.training.optim.accumulate_grad_batches=1 - - experiment.validation.batch_size=1 - - experiment.validation.limit_batch=1 - - experiment.validation.inference_mode=false - - experiment.validation.val_every_n_step=2000 - - experiment.training.checkpointing.every_n_train_steps=2000 - - experiment.training.checkpointing.save_last_k=5 - - experiment.training.max_steps=200000 - job: - name: main - chdir: null - override_dirname: +customized_load=true,+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors,+name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k,+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/,+seed=42,+seperate_load=true,+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors,algorithm.anchor_zero_geometry=false,algorithm.causal=true,algorithm.chunk_size=1,algorithm.context_frames=600,algorithm.diffusion.sampling_timesteps=20,algorithm.log_video=true,algorithm.memory_age.enabled=true,algorithm.memory_age.num_buckets=32,algorithm.memory_age.num_exact_buckets=17,algorithm.memory_age.overflow_start=100,algorithm.memory_attention_key_only_geometry=true,algorithm.memory_noise.enabled=true,algorithm.memory_noise.mode=source_free_logsnr,algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1,algorithm.memory_noise.stable_probability=0.25,algorithm.memory_noise.validation_noisy_memory=false,algorithm.memory_role_embedding=true,algorithm.memory_source_embedding=false,algorithm.memory_source_simulation.training_generated_suffix_max_frames=null,algorithm.memory_stream_count_correction=true,algorithm.metric_report_segment=100,algorithm.n_tokens=8,algorithm.noise_route.anchor=high,algorithm.noise_route.dynamic=low,algorithm.noise_route.revisit=all,algorithm.save_local=false,algorithm.trainability.full_dit_start_step=10000,algorithm.trainability.lr.base_dit=2.0e-5,algorithm.trainability.lr.memory_modules=8.0e-5,algorithm.trainability.train_full_dit=true,algorithm.use_memory_attention=true,algorithm.warmup_steps=500,algorithm=dememwm_base,auto_resume=true,dataset.context_length=100,dataset.memory_selection.causal=true,dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse,dataset.memory_selection.dynamic.selection_policy=multiview,dataset.memory_selection.enabled=true,dataset.memory_selection.max_anchor_frames=2,dataset.memory_selection.max_dynamic_frames=4,dataset.memory_selection.max_revisit_frames=2,dataset.memory_selection.training_dropout=0.1,dataset.n_frames=8,dataset.n_frames_valid=1100,dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features,dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft,dataset.single_eval_clip=true,dataset=video_minecraft_dememwm_latent,experiment.num_nodes=1,experiment.tasks=[training],experiment.training.batch_size=16,experiment.training.checkpointing.every_n_train_steps=2000,experiment.training.checkpointing.save_last_k=5,experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]},experiment.training.max_steps=200000,experiment.training.optim.accumulate_grad_batches=1,experiment.validation.batch_size=1,experiment.validation.inference_mode=false,experiment.validation.limit_batch=1,experiment.validation.val_every_n_step=2000,wandb.mode=offline - id: ??? - num: ??? - config_name: training - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.3' - cwd: /proj/cvl/users/x_fahkh2/DeMemWM - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /proj/cvl/users/x_fahkh2/DeMemWM/configurations - schema: file - provider: main - - path: '' - schema: structured - provider: schema - output_dir: /proj/cvl/users/x_fahkh2/DeMemWM/outputs/2026-07-18/00-46-54 - choices: - cluster: null - algorithm: dememwm_base - dataset: video_minecraft_dememwm_latent - experiment: exp_video - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2026-07-18/00-46-54/.hydra/overrides.yaml b/outputs/2026-07-18/00-46-54/.hydra/overrides.yaml deleted file mode 100644 index 56018cf8195f9be151815a3c8e6f6e4e8fa0508c..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-54/.hydra/overrides.yaml +++ /dev/null @@ -1,69 +0,0 @@ -- +name=train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k -- +output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/ -- +seed=42 -- auto_resume=true -- wandb.mode=offline -- experiment.tasks=[training] -- experiment.num_nodes=1 -- algorithm=dememwm_base -- +customized_load=true -- +seperate_load=true -- +diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors -- +vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors -- dataset=video_minecraft_dememwm_latent -- dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft -- dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features -- dataset.n_frames=8 -- dataset.n_frames_valid=1100 -- dataset.context_length=100 -- dataset.single_eval_clip=true -- dataset.memory_selection.enabled=true -- dataset.memory_selection.causal=true -- dataset.memory_selection.max_anchor_frames=2 -- dataset.memory_selection.max_dynamic_frames=4 -- dataset.memory_selection.max_revisit_frames=2 -- dataset.memory_selection.training_dropout=0.1 -- dataset.memory_selection.dynamic.selection_policy=multiview -- dataset.memory_selection.dynamic.multiview_selector=shared_area_diverse -- algorithm.causal=true -- algorithm.n_tokens=8 -- algorithm.context_frames=600 -- algorithm.metric_report_segment=100 -- algorithm.use_memory_attention=true -- algorithm.anchor_zero_geometry=false -- algorithm.memory_role_embedding=true -- algorithm.memory_source_embedding=false -- algorithm.memory_attention_key_only_geometry=true -- algorithm.memory_age.enabled=true -- algorithm.memory_age.num_buckets=32 -- algorithm.memory_age.num_exact_buckets=17 -- algorithm.memory_age.overflow_start=100 -- algorithm.memory_stream_count_correction=true -- algorithm.noise_route.anchor=high -- algorithm.noise_route.dynamic=low -- algorithm.noise_route.revisit=all -- algorithm.memory_source_simulation.training_generated_suffix_max_frames=null -- algorithm.memory_noise.enabled=true -- algorithm.memory_noise.mode=source_free_logsnr -- algorithm.memory_noise.stable_probability=0.25 -- algorithm.memory_noise.noisy_distribution=query_to_stabilization_uniform_logsnr_v1 -- algorithm.memory_noise.validation_noisy_memory=false -- algorithm.chunk_size=1 -- algorithm.warmup_steps=500 -- algorithm.trainability.train_full_dit=true -- algorithm.trainability.full_dit_start_step=10000 -- algorithm.trainability.lr.memory_modules=8.0e-5 -- algorithm.trainability.lr.base_dit=2.0e-5 -- algorithm.log_video=true -- algorithm.save_local=false -- algorithm.diffusion.sampling_timesteps=20 -- experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:200000,dataset:{wo_updown:false}}]} -- experiment.training.batch_size=16 -- experiment.training.optim.accumulate_grad_batches=1 -- experiment.validation.batch_size=1 -- experiment.validation.limit_batch=1 -- experiment.validation.inference_mode=false -- experiment.validation.val_every_n_step=2000 -- experiment.training.checkpointing.every_n_train_steps=2000 -- experiment.training.checkpointing.save_last_k=5 -- experiment.training.max_steps=200000 diff --git a/outputs/2026-07-18/00-46-54/main.log b/outputs/2026-07-18/00-46-54/main.log deleted file mode 100644 index 768b94577291d7d01d0c3e2addb2cd282534cd08..0000000000000000000000000000000000000000 --- a/outputs/2026-07-18/00-46-54/main.log +++ /dev/null @@ -1,51 +0,0 @@ -[2026-07-18 00:47:16,194][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 00:47:16,493][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 00:47:16,806][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 00:47:53,255][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 00:47:53,261][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 00:47:53,270][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 00:47:53,270][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 4, MEMBER: 5/8 -[2026-07-18 00:47:53,280][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 3, MEMBER: 4/8 -[2026-07-18 00:47:53,850][lightning.fabric.utilities.distributed][INFO] - Initializing distributed: GLOBAL_RANK: 6, MEMBER: 7/8 -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:16,106][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 00:48:19,590][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,590][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 00:48:19,590][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,545][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:04:40,544][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/amp.py:54: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. - -[2026-07-18 04:05:08,320][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 04:05:08,335][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 04:05:08,350][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 04:05:08,911][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,913][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:08,927][py.warnings][WARNING] - /proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/lightning/fabric/utilities/cloud_io.py:56: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. - -[2026-07-18 04:05:20,358][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,360][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:20,367][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 04:05:21,468][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:23,489][lightning.fabric.utilities.seed][INFO] - [rank: 6] Seed set to 42 -[2026-07-18 15:05:23,502][lightning.fabric.utilities.seed][INFO] - [rank: 4] Seed set to 42 -[2026-07-18 15:05:23,751][lightning.fabric.utilities.seed][INFO] - [rank: 3] Seed set to 42 -[2026-07-18 15:05:43,451][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 6 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,520][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 3 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:43,572][lightning.pytorch.accelerators.cuda][INFO] - LOCAL_RANK: 4 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-18 15:05:45,310][lightning.pytorch.trainer.connectors.signal_connector][INFO] - SLURM auto-requeueing enabled. Setting signal handlers. -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 6] Received SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 4] Received SIGTERM: 15 -[2026-07-21 00:47:00,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,366][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 -[2026-07-21 00:47:00,364][lightning.pytorch.trainer.connectors.signal_connector][INFO] - [rank: 3] Received SIGTERM: 15 -[2026-07-21 00:47:00,368][lightning.pytorch.trainer.connectors.signal_connector][INFO] - Bypassing SIGTERM: 15 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 948f09c36dda9f169153db5a3c8cc9411e7cb575..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06888499855995178,13.064892768859863,0.3279258608818054,800.0 -100,200,100,700,800,0.09095237404108047,11.103166580200195,0.4042089879512787,800.0 -200,300,100,800,900,0.09194152802228928,11.191773414611816,0.4079403579235077,800.0 -300,400,100,900,1000,0.11150308698415756,10.206729888916016,0.4254608154296875,800.0 -400,500,100,1000,1100,0.1316724568605423,9.617074012756348,0.4871872067451477,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index d87fe7fa208c09bdbb8c45ab5643dd7049e90123..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0707908645272255,12.939898490905762,0.3251693844795227,800.0 -100,200,100,700,800,0.08795671164989471,11.582192420959473,0.3870466649532318,800.0 -200,300,100,800,900,0.0785745307803154,11.921643257141113,0.3946800231933594,800.0 -300,400,100,900,1000,0.0796855166554451,11.816141128540039,0.4090360999107361,800.0 -400,500,100,1000,1100,0.08299916237592697,11.45152473449707,0.438315749168396,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 0c303b02897b36dbca806e6c717dbd8562aacc12..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06904318928718567,12.922369003295898,0.32136985659599304,800.0 -100,200,100,700,800,0.08130073547363281,11.481707572937012,0.3592129647731781,800.0 -200,300,100,800,900,0.0968020036816597,10.951699256896973,0.3738989233970642,800.0 -300,400,100,900,1000,0.08826474100351334,11.351846694946289,0.3613941967487335,800.0 -400,500,100,1000,1100,0.09351279586553574,10.973057746887207,0.4042759835720062,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 7f8f9cb8ed62b342a1de4fc603cae39aef4e0a6b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06341511756181717,13.278804779052734,0.3067120611667633,800.0 -100,200,100,700,800,0.09452704340219498,10.93684196472168,0.38173842430114746,800.0 -200,300,100,800,900,0.09074077755212784,10.940196990966797,0.3802950978279114,800.0 -300,400,100,900,1000,0.09236158430576324,10.880637168884277,0.38081419467926025,800.0 -400,500,100,1000,1100,0.09262711554765701,11.004284858703613,0.4004625082015991,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index fdef9fd3033a8e970ef3e2cab01abb1646af3488..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.059631358832120895,13.61989974975586,0.26630860567092896,800.0 -100,200,100,700,800,0.07443524152040482,11.899518966674805,0.3251277208328247,800.0 -200,300,100,800,900,0.07379640638828278,12.113446235656738,0.3322206139564514,800.0 -300,400,100,900,1000,0.07615838944911957,12.001047134399414,0.3289431631565094,800.0 -400,500,100,1000,1100,0.09206225723028183,10.910361289978027,0.38146254420280457,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index a5b39deee1daddf8e22f27a6af1e5dce008b6a24..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05403445288538933,13.98388671875,0.274010568857193,800.0 -100,200,100,700,800,0.07139523327350616,12.286293029785156,0.3291966915130615,800.0 -200,300,100,800,900,0.07828809320926666,11.764627456665039,0.3341537415981293,800.0 -300,400,100,900,1000,0.0811052992939949,11.72189712524414,0.32961195707321167,800.0 -400,500,100,1000,1100,0.0985068827867508,10.830940246582031,0.3929268717765808,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index 363dede4563063ea7cd9158b1530a83a9be84883..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.058318667113780975,14.086064338684082,0.24953234195709229,800.0 -100,200,100,700,800,0.07966641336679459,11.762557029724121,0.3113900125026703,800.0 -200,300,100,800,900,0.08700615912675858,11.399188041687012,0.3336807191371918,800.0 -300,400,100,900,1000,0.09091544896364212,11.272202491760254,0.31880447268486023,800.0 -400,500,100,1000,1100,0.10286402702331543,10.681203842163086,0.37326857447624207,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 2fe359bf31a93895e771d0c240a1121ffbd1428e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05304699018597603,14.431570053100586,0.23914991319179535,800.0 -100,200,100,700,800,0.06493135541677475,12.5538330078125,0.29864302277565,800.0 -200,300,100,800,900,0.06840861588716507,12.780810356140137,0.29617905616760254,800.0 -300,400,100,900,1000,0.06774236261844635,12.732670783996582,0.3012845516204834,800.0 -400,500,100,1000,1100,0.07807353883981705,11.980433464050293,0.3723520040512085,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index 38879532b82920a7039d41d136ce1f1baa917ac9..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04853038862347603,14.861541748046875,0.22376886010169983,800.0 -100,200,100,700,800,0.05675167590379715,13.453485488891602,0.26982787251472473,800.0 -200,300,100,800,900,0.07146239280700684,12.592765808105469,0.30213820934295654,800.0 -300,400,100,900,1000,0.06550553441047668,12.793254852294922,0.2884596884250641,800.0 -400,500,100,1000,1100,0.07504809647798538,12.090754508972168,0.34681105613708496,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 437760d8e473be7d3ba0805ee7bba85409f33571..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04874551296234131,14.865042686462402,0.21689635515213013,800.0 -100,200,100,700,800,0.06010834872722626,13.401711463928223,0.26659175753593445,800.0 -200,300,100,800,900,0.06527207046747208,13.21969223022461,0.27936115860939026,800.0 -300,400,100,900,1000,0.07456853985786438,12.192988395690918,0.31296882033348083,800.0 -400,500,100,1000,1100,0.08106296509504318,11.846379280090332,0.3594549596309662,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 3a1d1737e8bf6b1120a4b64257d23496a5cb1a56..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04789853096008301,14.634420394897461,0.2283961921930313,800.0 -100,200,100,700,800,0.06228318065404892,12.92959213256836,0.27230340242385864,800.0 -200,300,100,800,900,0.0701855719089508,12.737556457519531,0.28100916743278503,800.0 -300,400,100,900,1000,0.07838914543390274,12.347582817077637,0.3168458640575409,800.0 -400,500,100,1000,1100,0.09403546154499054,11.43383502960205,0.3924639821052551,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index a187ae69400690379b87b13e3800c21fe5f3ec5b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04235921427607536,15.3067626953125,0.2074817270040512,800.0 -100,200,100,700,800,0.05491309612989426,13.518646240234375,0.25284984707832336,800.0 -200,300,100,800,900,0.05581219121813774,13.782768249511719,0.2505562901496887,800.0 -300,400,100,900,1000,0.060826901346445084,13.090526580810547,0.27498704195022583,800.0 -400,500,100,1000,1100,0.0664597898721695,12.64074993133545,0.32792067527770996,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index a74372ee0fbf48bfdfd86ced34caaf45e53ce7c5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.046716056764125824,15.196752548217773,0.2047470211982727,800.0 -100,200,100,700,800,0.05002104863524437,13.811990737915039,0.23951362073421478,800.0 -200,300,100,800,900,0.06163500249385834,13.33201789855957,0.2694047689437866,800.0 -300,400,100,900,1000,0.057900696992874146,13.523116111755371,0.26414749026298523,800.0 -400,500,100,1000,1100,0.07248218357563019,12.735921859741211,0.30717960000038147,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index e5ee57421fe87edcc53d18e2c4c6d74b6d166e79..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03980649262666702,15.341673851013184,0.20309482514858246,800.0 -100,200,100,700,800,0.05148223787546158,13.658246994018555,0.25417548418045044,800.0 -200,300,100,800,900,0.05403036251664162,13.68086051940918,0.2600688934326172,800.0 -300,400,100,900,1000,0.05663047358393669,13.503545761108398,0.270294189453125,800.0 -400,500,100,1000,1100,0.06308063864707947,12.966200828552246,0.3166642487049103,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index b882ceb9b031a90c4d65bef473c0a52b1f3af2cd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12186145782470703,11.521646499633789,0.31610921025276184,800.0 -100,200,100,700,800,0.1638004183769226,10.462898254394531,0.40039002895355225,800.0 -200,300,100,800,900,0.14094744622707367,11.673105239868164,0.3332650661468506,800.0 -300,400,100,900,1000,0.17311036586761475,10.127041816711426,0.41348516941070557,800.0 -400,500,100,1000,1100,0.17752878367900848,10.132085800170898,0.414224237203598,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index 3af8148d7da54036e06193fb6ff6c7fa0d794853..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03336566314101219,19.57904624938965,0.19459940493106842,800.0 -100,200,100,700,800,0.056695904582738876,15.988838195800781,0.310754656791687,800.0 -200,300,100,800,900,0.05879715457558632,15.666145324707031,0.3078311085700989,800.0 -300,400,100,900,1000,0.08888742327690125,11.944375038146973,0.4150921702384949,800.0 -400,500,100,1000,1100,0.07287383824586868,13.913847923278809,0.3680918216705322,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index 12f4be51b31e0e0fdca800aef53d0fd4dd67a4b3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04008466750383377,19.215896606445312,0.200624480843544,800.0 -100,200,100,700,800,0.07016916573047638,15.636579513549805,0.27801713347435,800.0 -200,300,100,800,900,0.06218627095222473,18.012197494506836,0.29143136739730835,800.0 -300,400,100,900,1000,0.06767837703227997,14.16779899597168,0.31032827496528625,800.0 -400,500,100,1000,1100,0.06439635157585144,14.83418083190918,0.3270832896232605,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index d65ca4c59ea43cd7ba9beb3f7eb3ea5749d6ef47..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.028231268748641014,20.034523010253906,0.17448072135448456,800.0 -100,200,100,700,800,0.054470550268888474,16.92859649658203,0.26362812519073486,800.0 -200,300,100,800,900,0.05151268467307091,17.90257453918457,0.28251883387565613,800.0 -300,400,100,900,1000,0.08297282457351685,12.879212379455566,0.39062705636024475,800.0 -400,500,100,1000,1100,0.06123600900173187,16.300947189331055,0.35201266407966614,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index dc4eaf9923f178ccab6d62cec923bbac6f84b082..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03977835550904274,19.410276412963867,0.2239871472120285,800.0 -100,200,100,700,800,0.04854925721883774,17.371614456176758,0.276136189699173,800.0 -200,300,100,800,900,0.05023208633065224,18.864788055419922,0.2798241376876831,800.0 -300,400,100,900,1000,0.07110928744077682,13.633143424987793,0.35336852073669434,800.0 -400,500,100,1000,1100,0.04708399623632431,17.544466018676758,0.30620625615119934,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 35c7d98f13867a5d40f2f3b76bcbed5d8b78ca06..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.026242109015583992,20.90705108642578,0.16445480287075043,800.0 -100,200,100,700,800,0.03832745924592018,18.086904525756836,0.23752416670322418,800.0 -200,300,100,800,900,0.04375242069363594,19.035409927368164,0.26796624064445496,800.0 -300,400,100,900,1000,0.059180907905101776,14.741415023803711,0.3075486123561859,800.0 -400,500,100,1000,1100,0.03742426261305809,17.88154411315918,0.2587260901927948,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index f38486e4aea4964c147138d026ccfd9620aa42ae..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02449711784720421,21.21987533569336,0.1501040756702423,800.0 -100,200,100,700,800,0.04240470752120018,18.09173583984375,0.2209954857826233,800.0 -200,300,100,800,900,0.037998832762241364,19.455184936523438,0.2198360413312912,800.0 -300,400,100,900,1000,0.050287242978811264,15.275184631347656,0.24514488875865936,800.0 -400,500,100,1000,1100,0.037110891193151474,17.88347816467285,0.2418975681066513,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 16fcdff6040ee355fb171a44a8851aff322e9bb9..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03823951631784439,19.817594528198242,0.1825055480003357,800.0 -100,200,100,700,800,0.06431463360786438,16.2705020904541,0.28204143047332764,800.0 -200,300,100,800,900,0.05677936598658562,17.932987213134766,0.3072161376476288,800.0 -300,400,100,900,1000,0.07129930704832077,13.650476455688477,0.3505830764770508,800.0 -400,500,100,1000,1100,0.07087276875972748,15.981921195983887,0.3373350501060486,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 1cfbad85ac3c2d54db56c15148ce7cb38f9ea553..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.031506285071372986,20.19278907775879,0.17502029240131378,800.0 -100,200,100,700,800,0.0692557767033577,15.865108489990234,0.27591803669929504,800.0 -200,300,100,800,900,0.06701267510652542,16.470287322998047,0.3010390102863312,800.0 -300,400,100,900,1000,0.09975297749042511,12.467206001281738,0.3746800124645233,800.0 -400,500,100,1000,1100,0.05718091130256653,16.23236846923828,0.2939939796924591,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index 6bc56ccd695ebac00b3cfca25212daae158837e0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02920740284025669,20.751779556274414,0.18247535824775696,800.0 -100,200,100,700,800,0.055673204362392426,16.179418563842773,0.2836081087589264,800.0 -200,300,100,800,900,0.05344262719154358,17.7565860748291,0.2918911576271057,800.0 -300,400,100,900,1000,0.07568847388029099,13.4486722946167,0.37851738929748535,800.0 -400,500,100,1000,1100,0.04836387187242508,16.7941837310791,0.30576130747795105,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 6ee1d5795d8c6399a423f6afbf27962e5e8f9039..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024162190034985542,21.01129913330078,0.15038460493087769,800.0 -100,200,100,700,800,0.045069947838783264,17.247093200683594,0.2065623700618744,800.0 -200,300,100,800,900,0.06079521030187607,15.576995849609375,0.24346216022968292,800.0 -300,400,100,900,1000,0.07004707306623459,13.766371726989746,0.28694188594818115,800.0 -400,500,100,1000,1100,0.06731392443180084,15.156794548034668,0.29300472140312195,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index 6f826a97a42372f4d1d2da36ad853327b798f480..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03984290361404419,20.289962768554688,0.1841285675764084,800.0 -100,200,100,700,800,0.04926079884171486,17.593032836914062,0.2210892140865326,800.0 -200,300,100,800,900,0.04530799761414528,18.138320922851562,0.238557830452919,800.0 -300,400,100,900,1000,0.056275032460689545,14.614150047302246,0.305864542722702,800.0 -400,500,100,1000,1100,0.09811174124479294,16.18701934814453,0.306379497051239,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 827e8192ac54f2adba83531b1ba83d9ea438b0cc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.028489895164966583,20.660123825073242,0.16299766302108765,800.0 -100,200,100,700,800,0.0596148818731308,17.196874618530273,0.23407825827598572,800.0 -200,300,100,800,900,0.06389454007148743,17.889989852905273,0.25477877259254456,800.0 -300,400,100,900,1000,0.06583398580551147,14.774714469909668,0.2707865238189697,800.0 -400,500,100,1000,1100,0.04140488803386688,17.749217987060547,0.25501981377601624,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 942107eba739d8aa04dd70698edddd1ea16fb7e3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04732881113886833,19.048559188842773,0.2310492843389511,800.0 -100,200,100,700,800,0.07484769076108932,15.14999008178711,0.2899070978164673,800.0 -200,300,100,800,900,0.05476633086800575,17.07396697998047,0.28946229815483093,800.0 -300,400,100,900,1000,0.0730353444814682,12.865989685058594,0.35214102268218994,800.0 -400,500,100,1000,1100,0.06429767608642578,14.992609977722168,0.31582796573638916,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 295ab00089f0592345ee5a1a25dab1a93602d742..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02665363810956478,20.96402359008789,0.1550729125738144,800.0 -100,200,100,700,800,0.03603006899356842,18.296838760375977,0.20524704456329346,800.0 -200,300,100,800,900,0.040382515639066696,18.729076385498047,0.23122362792491913,800.0 -300,400,100,900,1000,0.05104931816458702,14.715288162231445,0.2637096643447876,800.0 -400,500,100,1000,1100,0.029732881113886833,18.72260284423828,0.2278079241514206,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index cef73963a8139986a3b0e64d2536380699feb175..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.032457511872053146,20.499910354614258,0.16559940576553345,800.0 -100,200,100,700,800,0.07116209715604782,16.010778427124023,0.2565852105617523,800.0 -200,300,100,800,900,0.04819774255156517,18.4073429107666,0.25918155908584595,800.0 -300,400,100,900,1000,0.05704038590192795,14.721644401550293,0.27444779872894287,800.0 -400,500,100,1000,1100,0.08681773394346237,14.933300971984863,0.30495885014533997,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 9ec5ff81420e2145c7d18283f8d315ce8e96667b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.034946050494909286,20.154861450195312,0.18813270330429077,800.0 -100,200,100,700,800,0.04335100203752518,17.809091567993164,0.2160096913576126,800.0 -200,300,100,800,900,0.051040973514318466,18.443944931030273,0.24023719131946564,800.0 -300,400,100,900,1000,0.04982022941112518,15.629388809204102,0.2490900605916977,800.0 -400,500,100,1000,1100,0.03620211035013199,18.009937286376953,0.23681770265102386,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index a4cf646e68108a1cb5e129f1096243f1c88d1066..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022830670699477196,21.541696548461914,0.13406039774417877,800.0 -100,200,100,700,800,0.03577042743563652,18.894411087036133,0.18403507769107819,800.0 -200,300,100,800,900,0.07311562448740005,17.92491912841797,0.24019207060337067,800.0 -300,400,100,900,1000,0.07344712316989899,14.788603782653809,0.25564083456993103,800.0 -400,500,100,1000,1100,0.11288949102163315,14.598467826843262,0.28663399815559387,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 09f06ad27c80828ed7c671770b3ef663c8fbd960..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0231753159314394,21.18982696533203,0.15095198154449463,800.0 -100,200,100,700,800,0.03127187862992287,18.967710494995117,0.1966315507888794,800.0 -200,300,100,800,900,0.03628668189048767,18.942190170288086,0.2151132971048355,800.0 -300,400,100,900,1000,0.041953034698963165,16.03803062438965,0.2088637351989746,800.0 -400,500,100,1000,1100,0.031584855169057846,18.0617733001709,0.22693032026290894,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 06e0c07340456b84c14164ac3c77d23c9b388946..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03234897181391716,20.06122398376465,0.19412679970264435,800.0 -100,200,100,700,800,0.038542430847883224,17.373092651367188,0.24620620906352997,800.0 -200,300,100,800,900,0.04973645135760307,18.224773406982422,0.27659872174263,800.0 -300,400,100,900,1000,0.06368595361709595,13.970501899719238,0.32080143690109253,800.0 -400,500,100,1000,1100,0.05445979908108711,16.037338256835938,0.2959727942943573,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 82628991f7dfb2254c6c5bb580d6b8f6516d3717..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02804923802614212,20.612398147583008,0.16140663623809814,800.0 -100,200,100,700,800,0.040183309465646744,18.017562866210938,0.1971728503704071,800.0 -200,300,100,800,900,0.048252176493406296,19.083520889282227,0.24439173936843872,800.0 -300,400,100,900,1000,0.07647807151079178,13.841692924499512,0.32812225818634033,800.0 -400,500,100,1000,1100,0.0649002194404602,16.770898818969727,0.2743421792984009,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index cb8cfbb0af3596148428b36e1f3484841d932340..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.045248717069625854,19.43003273010254,0.19879554212093353,800.0 -100,200,100,700,800,0.049956560134887695,17.265161514282227,0.25089776515960693,800.0 -200,300,100,800,900,0.05477543920278549,17.1049747467041,0.27250710129737854,800.0 -300,400,100,900,1000,0.0654936134815216,14.185001373291016,0.3020656108856201,800.0 -400,500,100,1000,1100,0.0427292063832283,16.25684356689453,0.2859628200531006,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index c57c2d71dd2a35008a47a123becd54b4c95ddf7c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023648858070373535,21.119836807250977,0.137422576546669,800.0 -100,200,100,700,800,0.04380817711353302,18.379833221435547,0.20138263702392578,800.0 -200,300,100,800,900,0.0455816425383091,18.623769760131836,0.23061975836753845,800.0 -300,400,100,900,1000,0.052343063056468964,15.164433479309082,0.24616752564907074,800.0 -400,500,100,1000,1100,0.03569114953279495,17.618452072143555,0.24569427967071533,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index ef2d9caf6dae4646abe126c279419eda0b7481cc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0288355965167284,20.675338745117188,0.1544816642999649,800.0 -100,200,100,700,800,0.05125078186392784,16.489971160888672,0.25333940982818604,800.0 -200,300,100,800,900,0.05083287134766579,17.825881958007812,0.25568073987960815,800.0 -300,400,100,900,1000,0.057084158062934875,14.161890983581543,0.28995123505592346,800.0 -400,500,100,1000,1100,0.045438170433044434,16.71137046813965,0.2747116684913635,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index 301051ad8dd22452a1ce496581e321a652f8cf45..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029769882559776306,20.567546844482422,0.15949216485023499,800.0 -100,200,100,700,800,0.03726442903280258,18.28012466430664,0.20873548090457916,800.0 -200,300,100,800,900,0.04185466468334198,18.945402145385742,0.2377854883670807,800.0 -300,400,100,900,1000,0.060511745512485504,14.110944747924805,0.3091951310634613,800.0 -400,500,100,1000,1100,0.047836169600486755,17.15985679626465,0.2457851618528366,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index 284cfdd601f6bb7b53987387fcd6eea0f99e68d6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05287415534257889,17.199010848999023,0.23897598683834076,800.0 -100,200,100,700,800,0.18212667107582092,10.557757377624512,0.4487367570400238,800.0 -200,300,100,800,900,0.27270784974098206,10.075248718261719,0.4908953607082367,800.0 -300,400,100,900,1000,0.3204394280910492,7.192131519317627,0.5532806515693665,800.0 -400,500,100,1000,1100,0.3715822696685791,7.796865940093994,0.4982835352420807,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index 4826fea6ccf60219459d4f8ad452efcaef967acb..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.052551835775375366,18.461803436279297,0.24753659963607788,800.0 -100,200,100,700,800,0.0863603949546814,14.367191314697266,0.34619757533073425,800.0 -200,300,100,800,900,0.10097727924585342,15.332797050476074,0.34264329075813293,800.0 -300,400,100,900,1000,0.09188957512378693,12.027957916259766,0.40637215971946716,800.0 -400,500,100,1000,1100,0.05362797901034355,15.383397102355957,0.3334827423095703,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 7048ff284e0fd0a75492433214ee20a4c6b229e0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.1054140254855156,14.453499794006348,0.3205481767654419,800.0 -100,200,100,700,800,0.20776979625225067,9.901214599609375,0.47503167390823364,800.0 -200,300,100,800,900,0.18124748766422272,12.129016876220703,0.4495846927165985,800.0 -300,400,100,900,1000,0.17985324561595917,9.34422779083252,0.5134319067001343,800.0 -400,500,100,1000,1100,0.23568275570869446,8.980088233947754,0.5073354840278625,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index 54a0f8d7aadc65f589890175b64b36c770f78cdc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06458216160535812,15.1920804977417,0.3550240695476532,800.0 -100,200,100,700,800,0.13330651819705963,9.967979431152344,0.5201969742774963,800.0 -200,300,100,800,900,0.15781524777412415,8.790687561035156,0.5333793759346008,800.0 -300,400,100,900,1000,0.17072954773902893,8.364148139953613,0.5979804992675781,800.0 -400,500,100,1000,1100,0.17673878371715546,8.547704696655273,0.5404525995254517,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index 6abdff518e3faf0113f5d4e054bb466b81c6f387..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12144002318382263,12.776141166687012,0.38927894830703735,800.0 -100,200,100,700,800,0.22086067497730255,7.237760543823242,0.5422203540802002,800.0 -200,300,100,800,900,0.1742304414510727,8.095293998718262,0.4301721155643463,800.0 -300,400,100,900,1000,0.15056580305099487,8.424345970153809,0.476826012134552,800.0 -400,500,100,1000,1100,0.1926417350769043,7.536762714385986,0.44176429510116577,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 31231f0843a876922795ab1f6fa058713e64b8b8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04896845668554306,18.51972198486328,0.2564898729324341,800.0 -100,200,100,700,800,0.054266124963760376,16.047895431518555,0.31984442472457886,800.0 -200,300,100,800,900,0.05376756563782692,16.814678192138672,0.30808696150779724,800.0 -300,400,100,900,1000,0.07014556229114532,13.00241470336914,0.39921778440475464,800.0 -400,500,100,1000,1100,0.06440550088882446,14.766051292419434,0.3424135148525238,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 8dccf6eaf5baa098608272ab92a5b1ac67076abd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07164738327264786,17.46368980407715,0.2854703962802887,800.0 -100,200,100,700,800,0.1384645700454712,11.851648330688477,0.43059420585632324,800.0 -200,300,100,800,900,0.1830407679080963,11.32606315612793,0.47912514209747314,800.0 -300,400,100,900,1000,0.13084211945533752,10.830754280090332,0.4771365821361542,800.0 -400,500,100,1000,1100,0.16546925902366638,10.629731178283691,0.4942517876625061,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 5b7424c034d97603f4b2fd30e745d72ac1010607..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04368835315108299,19.26565933227539,0.21736133098602295,800.0 -100,200,100,700,800,0.056242190301418304,15.325275421142578,0.30137112736701965,800.0 -200,300,100,800,900,0.09570884704589844,14.413788795471191,0.3477419316768646,800.0 -300,400,100,900,1000,0.07995850592851639,12.263453483581543,0.38417869806289673,800.0 -400,500,100,1000,1100,0.05727113410830498,14.845878601074219,0.3235926330089569,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index e95d049c8f8fef1f84bd8d621853c92b003e7d3a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05014592036604881,16.65041732788086,0.2671210765838623,800.0 -100,200,100,700,800,0.07175637036561966,12.624227523803711,0.3629106283187866,800.0 -200,300,100,800,900,0.06563770025968552,13.286565780639648,0.3317263126373291,800.0 -300,400,100,900,1000,0.09056428074836731,11.086752891540527,0.4182339608669281,800.0 -400,500,100,1000,1100,0.11035318672657013,11.34762191772461,0.3742162585258484,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index f62af02fbd13518b62e939770a9e043358fa9629..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.058928024023771286,18.168832778930664,0.25067102909088135,800.0 -100,200,100,700,800,0.12473712861537933,13.402996063232422,0.39413774013519287,800.0 -200,300,100,800,900,0.13933829963207245,13.238303184509277,0.41303810477256775,800.0 -300,400,100,900,1000,0.13649579882621765,10.299970626831055,0.46476635336875916,800.0 -400,500,100,1000,1100,0.21772147715091705,9.276555061340332,0.511486291885376,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index 1010c7cb3a82b1e67a0a8d15dead078a4d203626..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08614996075630188,14.581259727478027,0.33455416560173035,800.0 -100,200,100,700,800,0.1588401049375534,9.303487777709961,0.5218205451965332,800.0 -200,300,100,800,900,0.17082759737968445,9.061351776123047,0.5184818506240845,800.0 -300,400,100,900,1000,0.17710109055042267,8.2155122756958,0.5750316381454468,800.0 -400,500,100,1000,1100,0.24079813063144684,7.297722816467285,0.5718427896499634,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 0db83b315fb6f96a6338f00c6af75cbea724d118..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08006143569946289,15.201996803283691,0.24865585565567017,800.0 -100,200,100,700,800,0.1411670297384262,12.184955596923828,0.39183250069618225,800.0 -200,300,100,800,900,0.11993792653083801,13.21684455871582,0.37946200370788574,800.0 -300,400,100,900,1000,0.1091172993183136,11.249058723449707,0.4582323431968689,800.0 -400,500,100,1000,1100,0.11729634553194046,12.628386497497559,0.3781685531139374,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index 36bfda182e012ab1b97d4278da35021b3d5b70bb..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.041611701250076294,18.069595336914062,0.21043983101844788,800.0 -100,200,100,700,800,0.05214710533618927,16.2087459564209,0.2696099579334259,800.0 -200,300,100,800,900,0.05206330865621567,17.77989959716797,0.2610682547092438,800.0 -300,400,100,900,1000,0.06973784416913986,13.159526824951172,0.315656840801239,800.0 -400,500,100,1000,1100,0.04479098692536354,16.873371124267578,0.26149722933769226,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index ae8ec86bdec64b70279f85b21efc168251498e87..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.058627899736166,16.125520706176758,0.2515382468700409,800.0 -100,200,100,700,800,0.07043421268463135,14.558367729187012,0.33565565943717957,800.0 -200,300,100,800,900,0.07004162669181824,14.4049654006958,0.3214641809463501,800.0 -300,400,100,900,1000,0.1009102612733841,11.865073204040527,0.42245256900787354,800.0 -400,500,100,1000,1100,0.07998746633529663,13.258720397949219,0.3659057319164276,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index cf66915baa9027142e6f68c5a9a7f35a842cbe0d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06549689173698425,15.633625030517578,0.24145911633968353,800.0 -100,200,100,700,800,0.11455314606428146,12.43786334991455,0.3584420680999756,800.0 -200,300,100,800,900,0.11110074818134308,11.926369667053223,0.3723708987236023,800.0 -300,400,100,900,1000,0.09485277533531189,10.970453262329102,0.4076107144355774,800.0 -400,500,100,1000,1100,0.10882106423377991,11.00391960144043,0.381460964679718,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 82b05e2027d2a6290fa39cd07afdb6dfccc4bef5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06338925659656525,15.56347370147705,0.25576266646385193,800.0 -100,200,100,700,800,0.0906660258769989,12.18293571472168,0.3740498423576355,800.0 -200,300,100,800,900,0.09544249624013901,10.899497032165527,0.3637396991252899,800.0 -300,400,100,900,1000,0.0923500508069992,11.260026931762695,0.4125427305698395,800.0 -400,500,100,1000,1100,0.09918095916509628,12.218092918395996,0.3887277841567993,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 7717f9aa2743aac0f72b7cbbf1934ce2fff248d7..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04397882893681526,18.69661521911621,0.21688459813594818,800.0 -100,200,100,700,800,0.048290953040122986,16.47323226928711,0.27034273743629456,800.0 -200,300,100,800,900,0.04835636168718338,16.869190216064453,0.2761571705341339,800.0 -300,400,100,900,1000,0.07293517142534256,12.955649375915527,0.3429011404514313,800.0 -400,500,100,1000,1100,0.0416020005941391,17.482261657714844,0.281837522983551,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 200305fcd040acaa0a8f2f0ca20077329f3228b9..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04006266966462135,18.963401794433594,0.2241809219121933,800.0 -100,200,100,700,800,0.04495874047279358,17.055694580078125,0.2815471291542053,800.0 -200,300,100,800,900,0.048887502402067184,17.966083526611328,0.2844930589199066,800.0 -300,400,100,900,1000,0.06253154575824738,13.70152473449707,0.3608344793319702,800.0 -400,500,100,1000,1100,0.052167728543281555,15.812098503112793,0.3320469558238983,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 526d8b5f83d6d49d6168f3f928d301192ed76dd4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06663686037063599,15.865970611572266,0.258152574300766,800.0 -100,200,100,700,800,0.10132700949907303,12.56125259399414,0.3872259557247162,800.0 -200,300,100,800,900,0.11989199370145798,9.796282768249512,0.4145055413246155,800.0 -300,400,100,900,1000,0.14020301401615143,8.918789863586426,0.4888928234577179,800.0 -400,500,100,1000,1100,0.16071869432926178,9.516366958618164,0.4319401681423187,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 4b38c999ddbcfaee271b11d01765f5657d4b829e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.041443467140197754,18.90594482421875,0.20159912109375,800.0 -100,200,100,700,800,0.05960492044687271,15.600573539733887,0.29318535327911377,800.0 -200,300,100,800,900,0.055716924369335175,17.610984802246094,0.2822517454624176,800.0 -300,400,100,900,1000,0.07056319713592529,13.37759780883789,0.349247008562088,800.0 -400,500,100,1000,1100,0.046776361763477325,17.26287078857422,0.3023061752319336,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index 4ece08554c86dcb4ee802738d7b87a8ed3806402..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d0r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260703_183202-z7rcrdfj/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04858165234327316,18.861108779907227,0.24177256226539612,800.0 -100,200,100,700,800,0.057104453444480896,15.621759414672852,0.3046669363975525,800.0 -200,300,100,800,900,0.061047788709402084,17.044071197509766,0.31034427881240845,800.0 -300,400,100,900,1000,0.072824627161026,12.996883392333984,0.3691580295562744,800.0 -400,500,100,1000,1100,0.0451132208108902,16.95179557800293,0.3146475851535797,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 730206a12fbceba2ed613bffecfe411b98e4f6c2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06976467370986938,12.75132942199707,0.31580743193626404,800.0 -100,200,100,700,800,0.12294496595859528,9.576713562011719,0.4590849280357361,800.0 -200,300,100,800,900,0.12232984602451324,9.913141250610352,0.4440489113330841,800.0 -300,400,100,900,1000,0.11857765913009644,9.98205852508545,0.45228898525238037,800.0 -400,500,100,1000,1100,0.13311399519443512,9.309118270874023,0.4911854565143585,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index bb82c8a0b066c7cbb5b6ba0795be5c49d6710620..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06693270057439804,12.999702453613281,0.3047838509082794,800.0 -100,200,100,700,800,0.08276523649692535,11.661677360534668,0.3688540756702423,800.0 -200,300,100,800,900,0.07996074110269547,12.076133728027344,0.360361784696579,800.0 -300,400,100,900,1000,0.085358165204525,11.38166618347168,0.374245285987854,800.0 -400,500,100,1000,1100,0.09025007486343384,11.159398078918457,0.4110350012779236,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 7a8c1639abf4784c81e0db9344a34c0deb6301cc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07533338665962219,12.540035247802734,0.32385289669036865,800.0 -100,200,100,700,800,0.08109324425458908,11.565485000610352,0.3590913414955139,800.0 -200,300,100,800,900,0.08075982332229614,11.593001365661621,0.36279296875,800.0 -300,400,100,900,1000,0.07636874914169312,11.82797622680664,0.3720947206020355,800.0 -400,500,100,1000,1100,0.08833468705415726,10.871403694152832,0.41519486904144287,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index f152147eb3fc0cdfedf7bd246dfbbafe1b1ec848..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07155238837003708,12.653398513793945,0.305428683757782,800.0 -100,200,100,700,800,0.08365164697170258,11.45492172241211,0.3534608483314514,800.0 -200,300,100,800,900,0.08040092140436172,11.851672172546387,0.3563311696052551,800.0 -300,400,100,900,1000,0.08189035207033157,11.627386093139648,0.3706710934638977,800.0 -400,500,100,1000,1100,0.09407234191894531,11.001211166381836,0.4130491614341736,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 876065658f219ae732b6f70eb873493b6b994194..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07210345566272736,12.704399108886719,0.3185029625892639,800.0 -100,200,100,700,800,0.11105196177959442,10.304570198059082,0.4402886927127838,800.0 -200,300,100,800,900,0.09843342006206512,10.720978736877441,0.411368727684021,800.0 -300,400,100,900,1000,0.11131389439105988,10.352336883544922,0.41737717390060425,800.0 -400,500,100,1000,1100,0.12407241016626358,9.687740325927734,0.4784497022628784,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 1acffd53ba03b4ab573aaf51b9a5eefdff776830..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06480147689580917,12.779108047485352,0.3282894790172577,800.0 -100,200,100,700,800,0.07631200551986694,11.74020767211914,0.39381712675094604,800.0 -200,300,100,800,900,0.10130047053098679,10.551910400390625,0.45315903425216675,800.0 -300,400,100,900,1000,0.11602257937192917,10.540313720703125,0.44368958473205566,800.0 -400,500,100,1000,1100,0.12482883781194687,10.147762298583984,0.49514830112457275,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index becd4bf7ae4037ecfb779a4ba7a9ac99ea75ece5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07211732864379883,12.568172454833984,0.32309624552726746,800.0 -100,200,100,700,800,0.1113998144865036,9.939757347106934,0.44156554341316223,800.0 -200,300,100,800,900,0.11222679167985916,9.916274070739746,0.4186575412750244,800.0 -300,400,100,900,1000,0.11824682354927063,9.607280731201172,0.4551490843296051,800.0 -400,500,100,1000,1100,0.13136069476604462,9.120150566101074,0.49482908844947815,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 07144346f1f3f086983cf900fcfb49e73cc84d18..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.059539780020713806,13.508842468261719,0.2884787321090698,800.0 -100,200,100,700,800,0.07196077704429626,12.014178276062012,0.3519943356513977,800.0 -200,300,100,800,900,0.08397340029478073,11.490866661071777,0.36930298805236816,800.0 -300,400,100,900,1000,0.09411077201366425,11.069350242614746,0.3764512538909912,800.0 -400,500,100,1000,1100,0.11178885400295258,10.241942405700684,0.4330081045627594,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index ec825a1cc4349d949dedfc19ba2e03c2cd97e539..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05430877208709717,13.931217193603516,0.2630634009838104,800.0 -100,200,100,700,800,0.07457725703716278,12.042394638061523,0.32014334201812744,800.0 -200,300,100,800,900,0.08163172751665115,11.489099502563477,0.3422979712486267,800.0 -300,400,100,900,1000,0.07951154559850693,11.984256744384766,0.3276074230670929,800.0 -400,500,100,1000,1100,0.08712156116962433,11.474308013916016,0.38152778148651123,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 043b8b7551156a06f56148f72e6633b1f4dfac87..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05242845416069031,14.4976167678833,0.23605817556381226,800.0 -100,200,100,700,800,0.06876091659069061,12.383819580078125,0.30225518345832825,800.0 -200,300,100,800,900,0.07007846236228943,12.632245063781738,0.31958648562431335,800.0 -300,400,100,900,1000,0.08416251838207245,12.040599822998047,0.35865265130996704,800.0 -400,500,100,1000,1100,0.09213808923959732,11.415875434875488,0.414579838514328,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 3cb67b5a8902bbad75e89d75032a1a9382d82940..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05160810425877571,14.49919319152832,0.22753560543060303,800.0 -100,200,100,700,800,0.05823267996311188,13.162545204162598,0.2722100019454956,800.0 -200,300,100,800,900,0.06613748520612717,12.914855003356934,0.2786206305027008,800.0 -300,400,100,900,1000,0.06498567759990692,12.936716079711914,0.2874315679073334,800.0 -400,500,100,1000,1100,0.07411471009254456,12.149762153625488,0.34871214628219604,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index b9f53f0db39bf2cb2d4b068435d672bce7a5c535..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04320709407329559,15.20556640625,0.20753437280654907,800.0 -100,200,100,700,800,0.05183766409754753,13.695520401000977,0.24510376155376434,800.0 -200,300,100,800,900,0.056628890335559845,13.366995811462402,0.2650308310985565,800.0 -300,400,100,900,1000,0.0714745819568634,12.511195182800293,0.3210034966468811,800.0 -400,500,100,1000,1100,0.080619677901268,12.053452491760254,0.36179420351982117,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index 704669564fe501d613d4641e99ad0ac2dba30184..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.047368720173835754,14.793099403381348,0.22209011018276215,800.0 -100,200,100,700,800,0.053064629435539246,13.446882247924805,0.25922080874443054,800.0 -200,300,100,800,900,0.05945029854774475,13.610686302185059,0.26479271054267883,800.0 -300,400,100,900,1000,0.07377781718969345,12.235020637512207,0.3063899278640747,800.0 -400,500,100,1000,1100,0.07508032768964767,11.721108436584473,0.333009272813797,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 1644969ceb204215889c53373c3ad1cfcd40abbc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04066866263747215,15.330399513244629,0.20645426213741302,800.0 -100,200,100,700,800,0.05121401697397232,13.891199111938477,0.24947921931743622,800.0 -200,300,100,800,900,0.05645893141627312,13.830536842346191,0.2652357220649719,800.0 -300,400,100,900,1000,0.05464291200041771,13.78512954711914,0.26841437816619873,800.0 -400,500,100,1000,1100,0.0599391907453537,13.289379119873047,0.30506837368011475,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index faae459b530e33c39d769fef1ef7b9b1373e4829..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0908312052488327,12.522521018981934,0.35694611072540283,800.0 -100,200,100,700,800,0.1436970978975296,10.563414573669434,0.4761980473995209,800.0 -200,300,100,800,900,0.15803173184394836,9.202022552490234,0.48554474115371704,800.0 -300,400,100,900,1000,0.1763864904642105,8.283239364624023,0.542955756187439,800.0 -400,500,100,1000,1100,0.25541430711746216,6.608846664428711,0.632923424243927,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index 2aea722a63b46098bf68ba0b9083ba210d863087..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.030896196141839027,20.195966720581055,0.18992990255355835,800.0 -100,200,100,700,800,0.05107875168323517,16.555522918701172,0.27646300196647644,800.0 -200,300,100,800,900,0.04947759211063385,18.359830856323242,0.26063740253448486,800.0 -300,400,100,900,1000,0.0640997588634491,13.719513893127441,0.32593390345573425,800.0 -400,500,100,1000,1100,0.03584173694252968,18.102014541625977,0.2632366120815277,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index 5b0048885ee9c725dfa186653f250107061252a8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05603985860943794,18.41730499267578,0.24634471535682678,800.0 -100,200,100,700,800,0.0876966267824173,14.086247444152832,0.33972495794296265,800.0 -200,300,100,800,900,0.06798085570335388,16.591930389404297,0.304364949464798,800.0 -300,400,100,900,1000,0.09178619086742401,11.951726913452148,0.4146497845649719,800.0 -400,500,100,1000,1100,0.07191301882266998,15.190380096435547,0.3441189229488373,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index 5a2371ce769f26411aa3fe84fdeda416cdcbe70d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.037611111998558044,19.449867248535156,0.20050778985023499,800.0 -100,200,100,700,800,0.05613134801387787,17.137243270874023,0.27375948429107666,800.0 -200,300,100,800,900,0.057656921446323395,17.705673217773438,0.30335524678230286,800.0 -300,400,100,900,1000,0.0678318589925766,14.08605670928955,0.3353203535079956,800.0 -400,500,100,1000,1100,0.050160542130470276,16.653053283691406,0.30303290486335754,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 593311de0ec51556a96dda080276ff84fc6c5cc4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04621226340532303,19.615150451660156,0.21882545948028564,800.0 -100,200,100,700,800,0.04365139827132225,18.03569984436035,0.2251836359500885,800.0 -200,300,100,800,900,0.038130346685647964,19.34882354736328,0.23218326270580292,800.0 -300,400,100,900,1000,0.05651252344250679,15.255901336669922,0.2910812497138977,800.0 -400,500,100,1000,1100,0.034582797437906265,18.329065322875977,0.26368778944015503,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 0e91cb74b3ff1b9574b69bff0e3870755ebf0428..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02609887905418873,20.24869155883789,0.18303287029266357,800.0 -100,200,100,700,800,0.04676273837685585,17.566234588623047,0.25295570492744446,800.0 -200,300,100,800,900,0.054212894290685654,17.9423770904541,0.2930159270763397,800.0 -300,400,100,900,1000,0.0665617361664772,13.947880744934082,0.3472203314304352,800.0 -400,500,100,1000,1100,0.06828063726425171,15.06844711303711,0.35394248366355896,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 0c06e7776ce58cdea31b4c2f8193b5b562c2dd85..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0266026109457016,20.554458618164062,0.16974802315235138,800.0 -100,200,100,700,800,0.044682249426841736,17.988332748413086,0.238525390625,800.0 -200,300,100,800,900,0.055273786187171936,17.19951820373535,0.2863024175167084,800.0 -300,400,100,900,1000,0.0716727003455162,13.467460632324219,0.3273524343967438,800.0 -400,500,100,1000,1100,0.05861692503094673,15.717398643493652,0.30252277851104736,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 7d5e7a574ace1493101e98d15afc19bfdc684338..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029964342713356018,20.500377655029297,0.18200725317001343,800.0 -100,200,100,700,800,0.039970461279153824,18.224159240722656,0.22854959964752197,800.0 -200,300,100,800,900,0.048801206052303314,18.567550659179688,0.26464390754699707,800.0 -300,400,100,900,1000,0.072227843105793,13.604504585266113,0.33486053347587585,800.0 -400,500,100,1000,1100,0.05899495258927345,16.077312469482422,0.30865758657455444,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 7dffef91d2fdd0232518f28414ff29634d3c7618..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.048363570123910904,19.43009376525879,0.19380348920822144,800.0 -100,200,100,700,800,0.1011720597743988,14.531259536743164,0.2950008511543274,800.0 -200,300,100,800,900,0.11099965870380402,13.405743598937988,0.373085618019104,800.0 -300,400,100,900,1000,0.10857933759689331,12.041511535644531,0.3956722617149353,800.0 -400,500,100,1000,1100,0.134656623005867,12.39463996887207,0.4189586639404297,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index b1d4a6af4bafcdc1bdbb80d96bf8981f3b5adb13..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02543613500893116,21.121782302856445,0.1587725579738617,800.0 -100,200,100,700,800,0.03922927379608154,18.13224983215332,0.21616916358470917,800.0 -200,300,100,800,900,0.04020678624510765,19.344066619873047,0.23978090286254883,800.0 -300,400,100,900,1000,0.05684161186218262,14.758031845092773,0.2782405912876129,800.0 -400,500,100,1000,1100,0.045858632773160934,16.719106674194336,0.26812317967414856,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index ceafc1e86a3a1dac1ad3a36d22c1fbc09d351d45..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024451304227113724,20.935625076293945,0.1592274010181427,800.0 -100,200,100,700,800,0.041211746633052826,18.49204444885254,0.20043650269508362,800.0 -200,300,100,800,900,0.07335323095321655,16.80745506286621,0.2720329165458679,800.0 -300,400,100,900,1000,0.09657827019691467,12.812287330627441,0.3625672161579132,800.0 -400,500,100,1000,1100,0.18121691048145294,10.944531440734863,0.390116810798645,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index bf6b5b0bfc4164a120ded35f8619a14fe1b2203b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029581818729639053,20.371740341186523,0.15866458415985107,800.0 -100,200,100,700,800,0.05765356495976448,17.858131408691406,0.22596698999404907,800.0 -200,300,100,800,900,0.05178211256861687,18.755949020385742,0.2525252103805542,800.0 -300,400,100,900,1000,0.07115589827299118,13.58568000793457,0.3479398488998413,800.0 -400,500,100,1000,1100,0.08402903378009796,14.978799819946289,0.32125312089920044,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index d2f72aa96c88adc4ac8c74bcd13716f4c10c270c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04240032285451889,19.697555541992188,0.17699480056762695,800.0 -100,200,100,700,800,0.06547116488218307,15.756169319152832,0.24697652459144592,800.0 -200,300,100,800,900,0.06523892283439636,16.27012062072754,0.29525312781333923,800.0 -300,400,100,900,1000,0.0846538096666336,13.178193092346191,0.3471073806285858,800.0 -400,500,100,1000,1100,0.07509681582450867,14.511726379394531,0.3255103826522827,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index eaf7e5a07593ba32ef91d4fca1ce1091ed7f56cc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.026325764134526253,20.540719985961914,0.16448131203651428,800.0 -100,200,100,700,800,0.05895916000008583,16.970970153808594,0.24703602492809296,800.0 -200,300,100,800,900,0.08078864961862564,15.9569091796875,0.29016104340553284,800.0 -300,400,100,900,1000,0.07452674955129623,14.005359649658203,0.31132346391677856,800.0 -400,500,100,1000,1100,0.07553253322839737,15.283034324645996,0.31226909160614014,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 5383cd067e5c8573cf2b8a63a3fada96b131afbd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.020189516246318817,21.458820343017578,0.14696501195430756,800.0 -100,200,100,700,800,0.046626266092061996,18.49764633178711,0.20453862845897675,800.0 -200,300,100,800,900,0.06346146017313004,17.905553817749023,0.24546201527118683,800.0 -300,400,100,900,1000,0.06619129329919815,14.857939720153809,0.27895623445510864,800.0 -400,500,100,1000,1100,0.05437488108873367,17.8626708984375,0.26065030694007874,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 6fac8ed5d87826f9225c62c5d5085f66b5ee90d4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0315140001475811,20.394071578979492,0.17650657892227173,800.0 -100,200,100,700,800,0.044949110597372055,18.138900756835938,0.22526393830776215,800.0 -200,300,100,800,900,0.04451099783182144,18.972654342651367,0.2392367720603943,800.0 -300,400,100,900,1000,0.06112460792064667,14.06722640991211,0.28266090154647827,800.0 -400,500,100,1000,1100,0.03557842969894409,18.07493782043457,0.25047993659973145,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index ac40be3474ee6f18bc528959b503510bea418ec8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025139926001429558,20.8819580078125,0.16500583291053772,800.0 -100,200,100,700,800,0.042305465787649155,17.78983497619629,0.2103389948606491,800.0 -200,300,100,800,900,0.041342612355947495,18.802669525146484,0.21410885453224182,800.0 -300,400,100,900,1000,0.06864754855632782,14.604593276977539,0.28599560260772705,800.0 -400,500,100,1000,1100,0.04787499085068703,16.700809478759766,0.2515680193901062,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 203e8e639ce4fc29a9aefb265254fd657b247252..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025572801008820534,20.71228790283203,0.16284729540348053,800.0 -100,200,100,700,800,0.03829265758395195,18.106708526611328,0.20124730467796326,800.0 -200,300,100,800,900,0.06341836601495743,17.684616088867188,0.23770743608474731,800.0 -300,400,100,900,1000,0.05147701874375343,15.407509803771973,0.22674517333507538,800.0 -400,500,100,1000,1100,0.032666802406311035,18.30213165283203,0.21843911707401276,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index e4d1a7d86143aee38e50606b431e221cd9476b81..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.027551840990781784,20.628185272216797,0.1478576809167862,800.0 -100,200,100,700,800,0.05493279546499252,16.752273559570312,0.2670777440071106,800.0 -200,300,100,800,900,0.0419628731906414,19.05409049987793,0.2580842673778534,800.0 -300,400,100,900,1000,0.06603778898715973,14.183005332946777,0.3379833996295929,800.0 -400,500,100,1000,1100,0.03956811875104904,17.916576385498047,0.28363046050071716,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 04d15382bd2ecfd4a6865f3dab32952445005ae4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03002718836069107,20.614328384399414,0.1702919751405716,800.0 -100,200,100,700,800,0.030166342854499817,19.296079635620117,0.19052249193191528,800.0 -200,300,100,800,900,0.03502746671438217,19.0800838470459,0.21093600988388062,800.0 -300,400,100,900,1000,0.04618183523416519,15.781806945800781,0.2363496720790863,800.0 -400,500,100,1000,1100,0.02409312315285206,19.84807586669922,0.20795734226703644,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index b86cf728d983cf2550f71f0428a01a1f3fc13fd8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029131509363651276,20.90472412109375,0.1573261022567749,800.0 -100,200,100,700,800,0.04304826259613037,17.983991622924805,0.2282278835773468,800.0 -200,300,100,800,900,0.047212887555360794,18.406105041503906,0.25430965423583984,800.0 -300,400,100,900,1000,0.0632176399230957,14.145533561706543,0.29814016819000244,800.0 -400,500,100,1000,1100,0.02837359346449375,19.15095329284668,0.21582692861557007,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index c7614ff0eb017869ddcec12d14351a2d7c7fc833..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023687951266765594,20.69964599609375,0.1577686071395874,800.0 -100,200,100,700,800,0.03781862184405327,17.98621368408203,0.2319575548171997,800.0 -200,300,100,800,900,0.05832645297050476,15.749075889587402,0.28206706047058105,800.0 -300,400,100,900,1000,0.06750363856554031,13.503561973571777,0.3127612769603729,800.0 -400,500,100,1000,1100,0.06103069707751274,14.471322059631348,0.2995300590991974,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index c19c2e54c712497e3c60030b9263f5ed5efc0f8e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03244471549987793,20.12548828125,0.16764692962169647,800.0 -100,200,100,700,800,0.06783372163772583,16.030765533447266,0.24231275916099548,800.0 -200,300,100,800,900,0.10068372637033463,15.601953506469727,0.3091573715209961,800.0 -300,400,100,900,1000,0.09125974774360657,12.573874473571777,0.3473433554172516,800.0 -400,500,100,1000,1100,0.06359651684761047,14.5161714553833,0.3197856545448303,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 51f78b7efdcc5963748ec3d583f5eb089180745c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.028870966285467148,20.540565490722656,0.17709404230117798,800.0 -100,200,100,700,800,0.043446168303489685,18.186176300048828,0.23053817451000214,800.0 -200,300,100,800,900,0.04702748730778694,18.827987670898438,0.25192883610725403,800.0 -300,400,100,900,1000,0.07284156233072281,14.203141212463379,0.3272227942943573,800.0 -400,500,100,1000,1100,0.038781747221946716,17.858591079711914,0.2739887535572052,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index 9d8b55cf40fda26e41f05e7dbea93b18d073fcfc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029279591515660286,20.535398483276367,0.16430507600307465,800.0 -100,200,100,700,800,0.05005085468292236,17.49442481994629,0.24106726050376892,800.0 -200,300,100,800,900,0.04785456508398056,18.701377868652344,0.26852408051490784,800.0 -300,400,100,900,1000,0.07172375917434692,13.760427474975586,0.3434244990348816,800.0 -400,500,100,1000,1100,0.057486385107040405,16.31205177307129,0.31645798683166504,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index 313310d7c63ff8a71cbe3295cec4b9a737550904..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03289832919836044,20.472700119018555,0.15877749025821686,800.0 -100,200,100,700,800,0.07141231000423431,17.003803253173828,0.2395707666873932,800.0 -200,300,100,800,900,0.06170870363712311,17.650236129760742,0.2529434263706207,800.0 -300,400,100,900,1000,0.06024589389562607,15.412588119506836,0.24488991498947144,800.0 -400,500,100,1000,1100,0.03135809302330017,18.831584930419922,0.22397592663764954,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index 3767df62054f6c229cf332b508aec1ca30a0542b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022026846185326576,20.993091583251953,0.14220817387104034,800.0 -100,200,100,700,800,0.036311130970716476,18.172636032104492,0.2072777897119522,800.0 -200,300,100,800,900,0.03675677627325058,19.449275970458984,0.2250765562057495,800.0 -300,400,100,900,1000,0.04232388362288475,16.054405212402344,0.22181835770606995,800.0 -400,500,100,1000,1100,0.032898642122745514,18.59829330444336,0.23087884485721588,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 5f690c054cc46be1d00f12b5d6a9d60fc864e19e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.036416247487068176,19.613929748535156,0.18357500433921814,800.0 -100,200,100,700,800,0.04665838181972504,17.40671157836914,0.23993617296218872,800.0 -200,300,100,800,900,0.04911551624536514,17.994962692260742,0.26270079612731934,800.0 -300,400,100,900,1000,0.058766257017850876,14.786957740783691,0.2938087582588196,800.0 -400,500,100,1000,1100,0.03569061681628227,17.64377784729004,0.2555294930934906,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index 86373cae49136ffab5dc94c895c1bffe23cc909e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022654738277196884,21.331106185913086,0.13357196748256683,800.0 -100,200,100,700,800,0.10066068917512894,16.113018035888672,0.2799132466316223,800.0 -200,300,100,800,900,0.1141582503914833,15.5305757522583,0.3249236047267914,800.0 -300,400,100,900,1000,0.15090404450893402,12.339241027832031,0.3801935315132141,800.0 -400,500,100,1000,1100,0.19086165726184845,12.175956726074219,0.380909264087677,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index c01d238be2f9732a9b5dd8628c16d55d4954f9f4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03786628693342209,20.977088928222656,0.15949195623397827,800.0 -100,200,100,700,800,0.0890217050909996,17.15782928466797,0.24231502413749695,800.0 -200,300,100,800,900,0.08985263854265213,16.797819137573242,0.27375149726867676,800.0 -300,400,100,900,1000,0.08275877684354782,14.706014633178711,0.27799084782600403,800.0 -400,500,100,1000,1100,0.1063501164317131,15.799525260925293,0.2887688875198364,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index c1f0fe6cf064593de971967f767afe7cb0eda5c4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02218547835946083,21.224618911743164,0.1526084840297699,800.0 -100,200,100,700,800,0.04017040878534317,18.67627716064453,0.20558655261993408,800.0 -200,300,100,800,900,0.049029815942049026,19.05162811279297,0.25205186009407043,800.0 -300,400,100,900,1000,0.05812208354473114,15.061595916748047,0.28139451146125793,800.0 -400,500,100,1000,1100,0.03991616889834404,17.889751434326172,0.2603461742401123,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 29576d220bf31380d068a3912030337771015173..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.038620173931121826,20.184606552124023,0.1684218943119049,800.0 -100,200,100,700,800,0.07759415358304977,16.852100372314453,0.23902109265327454,800.0 -200,300,100,800,900,0.053700342774391174,17.936491012573242,0.24157901108264923,800.0 -300,400,100,900,1000,0.07570422440767288,14.88739013671875,0.2734186053276062,800.0 -400,500,100,1000,1100,0.09033741056919098,16.051177978515625,0.27330100536346436,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 5536e4a54b0761da1806fa0f857e872c949abfd1..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.030759839341044426,20.862329483032227,0.16253653168678284,800.0 -100,200,100,700,800,0.042944759130477905,17.430505752563477,0.23683856427669525,800.0 -200,300,100,800,900,0.047898389399051666,18.643310546875,0.2647933065891266,800.0 -300,400,100,900,1000,0.06755337864160538,14.17715835571289,0.30398231744766235,800.0 -400,500,100,1000,1100,0.05884936451911926,16.109691619873047,0.30688008666038513,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index 364da38967d6b06aa60f9deccd4bb5025ad4dfb7..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02181556634604931,21.5142879486084,0.14728587865829468,800.0 -100,200,100,700,800,0.03837994113564491,18.548063278198242,0.20155389606952667,800.0 -200,300,100,800,900,0.0469040721654892,19.056474685668945,0.2204713076353073,800.0 -300,400,100,900,1000,0.05119263008236885,15.567849159240723,0.24616672098636627,800.0 -400,500,100,1000,1100,0.03951853886246681,17.78321647644043,0.23409169912338257,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index b5a4d145c8e312d02678d5bcccb615174f8ddee3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022986549884080887,20.7772159576416,0.15740829706192017,800.0 -100,200,100,700,800,0.04382271319627762,18.285850524902344,0.20932535827159882,800.0 -200,300,100,800,900,0.04478592798113823,18.787389755249023,0.23114170134067535,800.0 -300,400,100,900,1000,0.07018180936574936,14.414384841918945,0.2858850359916687,800.0 -400,500,100,1000,1100,0.08666200935840607,14.80632209777832,0.30919140577316284,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index 051eaedc441a781ed31f14f27121224ef2e1d069..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022534556686878204,21.099145889282227,0.13456834852695465,800.0 -100,200,100,700,800,0.03885112702846527,18.307159423828125,0.1959824562072754,800.0 -200,300,100,800,900,0.039029765874147415,19.358793258666992,0.22580796480178833,800.0 -300,400,100,900,1000,0.05712679773569107,15.102958679199219,0.2829614579677582,800.0 -400,500,100,1000,1100,0.032061416655778885,18.876310348510742,0.2449515163898468,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 0275ac50ffbf53ebb941dc14dbdb54b83d2c9977..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024896960705518723,20.83704376220703,0.1424976885318756,800.0 -100,200,100,700,800,0.040909890085458755,18.24594497680664,0.1979069858789444,800.0 -200,300,100,800,900,0.04105393588542938,19.15007972717285,0.21388639509677887,800.0 -300,400,100,900,1000,0.04759686440229416,15.77905559539795,0.22737501561641693,800.0 -400,500,100,1000,1100,0.04507485032081604,17.852832794189453,0.2266256958246231,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index e90b07cdbb6530801f725edb402c59995b4a2be0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024597177281975746,20.954486846923828,0.14238420128822327,800.0 -100,200,100,700,800,0.04948309063911438,17.61786460876465,0.21144771575927734,800.0 -200,300,100,800,900,0.05060422793030739,17.146717071533203,0.2511174976825714,800.0 -300,400,100,900,1000,0.0818532258272171,13.57571029663086,0.3456369638442993,800.0 -400,500,100,1000,1100,0.04338115081191063,16.9753360748291,0.27344679832458496,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index 1e8fc24791075f21281be7e76bdc5247ce258acc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02889392152428627,20.50591278076172,0.15922315418720245,800.0 -100,200,100,700,800,0.046578340232372284,17.53325653076172,0.22077296674251556,800.0 -200,300,100,800,900,0.03743014112114906,19.60633087158203,0.2161622941493988,800.0 -300,400,100,900,1000,0.04810507595539093,15.332468032836914,0.24426595866680145,800.0 -400,500,100,1000,1100,0.07143428921699524,16.11495590209961,0.2623221278190613,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index 8d9970c489208d09ab6e76f1a5ffdcd303b0a0ef..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02491002529859543,21.119775772094727,0.15889407694339752,800.0 -100,200,100,700,800,0.04894239455461502,18.575414657592773,0.1928020417690277,800.0 -200,300,100,800,900,0.042885590344667435,19.221790313720703,0.2013063281774521,800.0 -300,400,100,900,1000,0.04834276810288429,15.85571002960205,0.21583010256290436,800.0 -400,500,100,1000,1100,0.03115518018603325,18.796674728393555,0.21121767163276672,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index c13dff338bd672a97090e8cabb76cd3b82296213..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03341423347592354,20.486623764038086,0.14965850114822388,800.0 -100,200,100,700,800,0.07933030277490616,16.552474975585938,0.23188701272010803,800.0 -200,300,100,800,900,0.09041006863117218,17.267114639282227,0.2593986988067627,800.0 -300,400,100,900,1000,0.06515618413686752,14.845220565795898,0.2661176919937134,800.0 -400,500,100,1000,1100,0.08185624331235886,16.280397415161133,0.27774903178215027,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 7b69259ebd0ca2fca924e38ccc0da5716394ea62..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.027414457872509956,20.785249710083008,0.1620001345872879,800.0 -100,200,100,700,800,0.043554067611694336,17.669992446899414,0.22154289484024048,800.0 -200,300,100,800,900,0.04524308815598488,18.731948852539062,0.24608388543128967,800.0 -300,400,100,900,1000,0.06436078250408173,14.46683120727539,0.3069800138473511,800.0 -400,500,100,1000,1100,0.04016423225402832,17.477252960205078,0.2920874059200287,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 1c92a0f0867bec283f4730d74d4141ff78f3db99..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024405574426054955,20.814918518066406,0.14493146538734436,800.0 -100,200,100,700,800,0.055918049067258835,16.805763244628906,0.2488681823015213,800.0 -200,300,100,800,900,0.09462223201990128,15.810962677001953,0.29456156492233276,800.0 -300,400,100,900,1000,0.09505516290664673,12.955671310424805,0.34035807847976685,800.0 -400,500,100,1000,1100,0.0920766294002533,15.542139053344727,0.3071753978729248,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 0fe4326fd277248e8750e1be7650d4a4ae1f13bd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.030639931559562683,20.592424392700195,0.16933971643447876,800.0 -100,200,100,700,800,0.05608167126774788,16.447114944458008,0.24983276426792145,800.0 -200,300,100,800,900,0.045436982065439224,17.618192672729492,0.23563174903392792,800.0 -300,400,100,900,1000,0.08323311060667038,13.3046236038208,0.3177529573440552,800.0 -400,500,100,1000,1100,0.07057765126228333,14.826526641845703,0.29020586609840393,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 5b8d1d37e18dc337506dd656cc980071d52cc509..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.030744509771466255,20.732406616210938,0.15181046724319458,800.0 -100,200,100,700,800,0.0538412444293499,17.311281204223633,0.22924233973026276,800.0 -200,300,100,800,900,0.07625885307788849,17.946008682250977,0.2592986226081848,800.0 -300,400,100,900,1000,0.0723329558968544,14.632006645202637,0.28313538432121277,800.0 -400,500,100,1000,1100,0.07231085002422333,15.957635879516602,0.312921404838562,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index efab39641e34354e539115609a24e741d444482a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06484078615903854,17.39698028564453,0.25935783982276917,800.0 -100,200,100,700,800,0.1245875135064125,10.056676864624023,0.42116260528564453,800.0 -200,300,100,800,900,0.10472045093774796,10.174858093261719,0.4160831570625305,800.0 -300,400,100,900,1000,0.11425943672657013,9.63364315032959,0.5054700970649719,800.0 -400,500,100,1000,1100,0.10099321603775024,10.399758338928223,0.442950963973999,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index cb02f3f4740cd836a61b8bca4e6aec8cd9d6b911..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04609805718064308,18.944171905517578,0.21370640397071838,800.0 -100,200,100,700,800,0.0592348575592041,15.496543884277344,0.29255062341690063,800.0 -200,300,100,800,900,0.05131728947162628,17.368288040161133,0.26384997367858887,800.0 -300,400,100,900,1000,0.07218292355537415,13.10116195678711,0.3377452790737152,800.0 -400,500,100,1000,1100,0.05182211101055145,15.949270248413086,0.28912174701690674,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index 43b50cab194fb25defe4bbcc24fe4ba24e8d256f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04798883572220802,19.27664566040039,0.18912674486637115,800.0 -100,200,100,700,800,0.09984225034713745,15.5224027633667,0.2882876694202423,800.0 -200,300,100,800,900,0.08419500291347504,16.690563201904297,0.2946726083755493,800.0 -300,400,100,900,1000,0.08429595828056335,13.365460395812988,0.30712518095970154,800.0 -400,500,100,1000,1100,0.06306514143943787,15.528278350830078,0.2879483699798584,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index f3049836ec0aba13e82b07d820e2f1b7ba0db990..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06339807063341141,17.20220375061035,0.27029454708099365,800.0 -100,200,100,700,800,0.14315830171108246,11.311662673950195,0.41331642866134644,800.0 -200,300,100,800,900,0.21390199661254883,9.510069847106934,0.4772219955921173,800.0 -300,400,100,900,1000,0.1838666945695877,8.326676368713379,0.5282676219940186,800.0 -400,500,100,1000,1100,0.261698842048645,6.977132797241211,0.5451915264129639,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index cccc047c5c2ca39b24d8eee8ca03b559a09a1384..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04411951079964638,18.74943733215332,0.22872592508792877,800.0 -100,200,100,700,800,0.048910725861787796,16.315176010131836,0.2987467646598816,800.0 -200,300,100,800,900,0.047495465725660324,17.5786190032959,0.29100126028060913,800.0 -300,400,100,900,1000,0.059753939509391785,13.954449653625488,0.36397480964660645,800.0 -400,500,100,1000,1100,0.0414554737508297,16.419431686401367,0.31656894087791443,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index abe1c16e91be835eabb202ac2b555a0be7ffb012..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.043153341859579086,19.711061477661133,0.22313529253005981,800.0 -100,200,100,700,800,0.04349623620510101,17.79395866394043,0.2901083528995514,800.0 -200,300,100,800,900,0.05287669226527214,18.278343200683594,0.2926156520843506,800.0 -300,400,100,900,1000,0.09442450851202011,12.945117950439453,0.40317171812057495,800.0 -400,500,100,1000,1100,0.10745057463645935,15.755898475646973,0.3745320439338684,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index cf6cf97d6f5fe114b56770e17eed94084c1b9216..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.13339819014072418,12.449470520019531,0.4301268458366394,800.0 -100,200,100,700,800,0.29475855827331543,5.782345771789551,0.6916735768318176,800.0 -200,300,100,800,900,0.2915199398994446,5.547295570373535,0.7058871388435364,800.0 -300,400,100,900,1000,0.3319656252861023,4.880416393280029,0.7808993458747864,800.0 -400,500,100,1000,1100,0.38167205452919006,4.269643306732178,0.7690334916114807,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index 9b06536810c765309a24644202d842bc3912bb4d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04586773365736008,19.269224166870117,0.2586607038974762,800.0 -100,200,100,700,800,0.07795719802379608,15.940338134765625,0.33104369044303894,800.0 -200,300,100,800,900,0.0763922855257988,17.226394653320312,0.3148733973503113,800.0 -300,400,100,900,1000,0.08881477266550064,13.087834358215332,0.3716960847377777,800.0 -400,500,100,1000,1100,0.10814152657985687,15.177850723266602,0.35391587018966675,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index 0080b072ee97dd1aa89526bd7a66261570c90f8a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05026655271649361,18.360370635986328,0.242411270737648,800.0 -100,200,100,700,800,0.08040544390678406,14.824378967285156,0.33146151900291443,800.0 -200,300,100,800,900,0.08930423855781555,14.966706275939941,0.3323478698730469,800.0 -300,400,100,900,1000,0.0902424156665802,12.17941665649414,0.3965286612510681,800.0 -400,500,100,1000,1100,0.08625984936952591,13.979900360107422,0.34187307953834534,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index 5bae84676c67236a5c79869bd8c647cd427c2c1a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.053136661648750305,18.281766891479492,0.2316727191209793,800.0 -100,200,100,700,800,0.09452784806489944,14.156014442443848,0.3364667594432831,800.0 -200,300,100,800,900,0.10357090085744858,15.387602806091309,0.34724757075309753,800.0 -300,400,100,900,1000,0.1060015857219696,11.112499237060547,0.45210447907447815,800.0 -400,500,100,1000,1100,0.12580962479114532,12.712389945983887,0.3808992803096771,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 807628825133a7dc9532b1408a9b6402ace4009f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05068770423531532,19.121532440185547,0.19684723019599915,800.0 -100,200,100,700,800,0.10662306100130081,13.554983139038086,0.319852352142334,800.0 -200,300,100,800,900,0.11886724829673767,14.384681701660156,0.3330761790275574,800.0 -300,400,100,900,1000,0.1309751570224762,10.963399887084961,0.4121720790863037,800.0 -400,500,100,1000,1100,0.20215004682540894,9.563238143920898,0.43907487392425537,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index 5b8bd940ef9456921c6723714d44fe4f4c29577e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.036164455115795135,19.90552520751953,0.18020030856132507,800.0 -100,200,100,700,800,0.07338325679302216,16.274452209472656,0.25784537196159363,800.0 -200,300,100,800,900,0.06038280948996544,17.44660186767578,0.27538326382637024,800.0 -300,400,100,900,1000,0.054517846554517746,14.41382122039795,0.2872093915939331,800.0 -400,500,100,1000,1100,0.04099278524518013,17.439931869506836,0.24748340249061584,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index 69326367f3f441a848f9cfb32764f383808e84fb..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.053304024040699005,17.410192489624023,0.2882511615753174,800.0 -100,200,100,700,800,0.08969110250473022,12.632308006286621,0.3979821801185608,800.0 -200,300,100,800,900,0.0705990120768547,15.136758804321289,0.35059523582458496,800.0 -300,400,100,900,1000,0.09682686626911163,11.447210311889648,0.44865965843200684,800.0 -400,500,100,1000,1100,0.09412530809640884,12.049599647521973,0.4443342983722687,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index 9397aa20c441d1fc401913a13b148d366efbf0f2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04031044989824295,19.750730514526367,0.17884418368339539,800.0 -100,200,100,700,800,0.07920247316360474,15.987103462219238,0.26403436064720154,800.0 -200,300,100,800,900,0.08286981284618378,17.026206970214844,0.2843683660030365,800.0 -300,400,100,900,1000,0.07918813824653625,13.592094421386719,0.3344118595123291,800.0 -400,500,100,1000,1100,0.105133056640625,14.159692764282227,0.3364603519439697,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index 1d0df2f7543a9d6677fae9a1ea1b124429ca664b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02958034910261631,20.10491180419922,0.18784336745738983,800.0 -100,200,100,700,800,0.04820983111858368,16.710559844970703,0.29135096073150635,800.0 -200,300,100,800,900,0.0439649373292923,17.420087814331055,0.31274038553237915,800.0 -300,400,100,900,1000,0.060056768357753754,13.278328895568848,0.4053734540939331,800.0 -400,500,100,1000,1100,0.05847776308655739,14.121766090393066,0.3885020315647125,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index c9f45796a16f07e243467f5496ba171cd101fd90..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_causal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192406-rodb8uhx/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.040465421974658966,19.553464889526367,0.20082515478134155,800.0 -100,200,100,700,800,0.054553862661123276,15.793432235717773,0.31343939900398254,800.0 -200,300,100,800,900,0.06183256953954697,16.090038299560547,0.3291130065917969,800.0 -300,400,100,900,1000,0.09375501424074173,12.015629768371582,0.4155687689781189,800.0 -400,500,100,1000,1100,0.11973116546869278,13.459157943725586,0.4255467653274536,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index b1824822294b28598f83dbcddd4271ca627e5ef1..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07851789891719818,12.173672676086426,0.344931423664093,800.0 -100,200,100,700,800,0.09736734628677368,10.519372940063477,0.40608230233192444,800.0 -200,300,100,800,900,0.09490399062633514,10.866693496704102,0.4170895516872406,800.0 -300,400,100,900,1000,0.09284944832324982,11.027130126953125,0.43576353788375854,800.0 -400,500,100,1000,1100,0.10656558722257614,10.378268241882324,0.4897283911705017,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index d1c785ac154e5e2701575197ad94ccea2b0d8f01..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07778807729482651,12.396941184997559,0.3295651376247406,800.0 -100,200,100,700,800,0.08763886243104935,11.267126083374023,0.36370790004730225,800.0 -200,300,100,800,900,0.08215440064668655,11.801076889038086,0.3651143014431,800.0 -300,400,100,900,1000,0.08436586707830429,11.41724967956543,0.3728939890861511,800.0 -400,500,100,1000,1100,0.10146533697843552,10.355700492858887,0.4343340992927551,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 7f556df2d7a7a9c02af972df24b62d24a64ec5ce..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07731159031391144,12.297654151916504,0.3280915915966034,800.0 -100,200,100,700,800,0.10526877641677856,10.258622169494629,0.40673431754112244,800.0 -200,300,100,800,900,0.11807757616043091,9.8528470993042,0.4564642310142517,800.0 -300,400,100,900,1000,0.11319415271282196,10.300764083862305,0.4321354627609253,800.0 -400,500,100,1000,1100,0.10733550786972046,10.223482131958008,0.4422529637813568,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index b32635ab25f3139e3dcc7597a7623cc08e49dbd4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07677389681339264,12.296357154846191,0.3334348201751709,800.0 -100,200,100,700,800,0.08810510486364365,11.123024940490723,0.38268235325813293,800.0 -200,300,100,800,900,0.09299832582473755,10.990697860717773,0.3957628011703491,800.0 -300,400,100,900,1000,0.10764827579259872,10.351449012756348,0.41888609528541565,800.0 -400,500,100,1000,1100,0.11936946213245392,9.803793907165527,0.4644300937652588,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 2e0eb074167c2b343743eada458c7025eb9f6c83..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06762921810150146,12.910749435424805,0.3162182867527008,800.0 -100,200,100,700,800,0.07792209833860397,11.708293914794922,0.37402573227882385,800.0 -200,300,100,800,900,0.09968648850917816,10.901165962219238,0.3897486925125122,800.0 -300,400,100,900,1000,0.09121374040842056,11.38162612915039,0.37687164545059204,800.0 -400,500,100,1000,1100,0.10144975781440735,10.888628959655762,0.4268859922885895,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 5cc3f8b508a2bbbd1fae883a12b8f08496e0805d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0754694864153862,12.440942764282227,0.33118247985839844,800.0 -100,200,100,700,800,0.09275317937135696,10.882380485534668,0.41517943143844604,800.0 -200,300,100,800,900,0.11426129937171936,10.165024757385254,0.450621634721756,800.0 -300,400,100,900,1000,0.10727840662002563,10.542415618896484,0.43004775047302246,800.0 -400,500,100,1000,1100,0.12032457441091537,9.831233978271484,0.46988892555236816,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index d85f0328d8d14657126ac667a89db58483a8c182..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06399703025817871,13.11518669128418,0.2944011092185974,800.0 -100,200,100,700,800,0.07424803823232651,11.851113319396973,0.3449215590953827,800.0 -200,300,100,800,900,0.08377806842327118,11.31360149383545,0.36755722761154175,800.0 -300,400,100,900,1000,0.09255879372358322,10.866167068481445,0.37550467252731323,800.0 -400,500,100,1000,1100,0.09162034094333649,11.013359069824219,0.4034268260002136,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 1fe3ff11d80ba238fe0f46824db98d4d8451711d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06949948519468307,13.043893814086914,0.30282512307167053,800.0 -100,200,100,700,800,0.08575523644685745,11.475833892822266,0.3696533143520355,800.0 -200,300,100,800,900,0.10133779793977737,10.783039093017578,0.39837172627449036,800.0 -300,400,100,900,1000,0.11151691526174545,10.28299331665039,0.41688355803489685,800.0 -400,500,100,1000,1100,0.11076487600803375,10.228399276733398,0.45538589358329773,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index a8ede8a7c2e7f3cfafd9335955bac8612db93c01..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06066155806183815,13.467555046081543,0.29065462946891785,800.0 -100,200,100,700,800,0.0764477327466011,11.619885444641113,0.3566819727420807,800.0 -200,300,100,800,900,0.07717261463403702,11.795110702514648,0.36874908208847046,800.0 -300,400,100,900,1000,0.08898108452558517,11.049485206604004,0.3853437900543213,800.0 -400,500,100,1000,1100,0.09053876250982285,11.009430885314941,0.4180612862110138,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index c984ae177c7791252cd37ecc6b22cf7def7b2883..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05885465443134308,13.967467308044434,0.2681751251220703,800.0 -100,200,100,700,800,0.06928715109825134,12.238424301147461,0.31440475583076477,800.0 -200,300,100,800,900,0.0754740759730339,11.774954795837402,0.34039658308029175,800.0 -300,400,100,900,1000,0.08223559707403183,11.50167465209961,0.34049302339553833,800.0 -400,500,100,1000,1100,0.09193094074726105,11.16296100616455,0.3813732862472534,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 7a079ca9079b6c8fa1edbb4e20b16c11c2377725..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05620531737804413,14.102985382080078,0.2569023072719574,800.0 -100,200,100,700,800,0.06474155187606812,12.52859878540039,0.31771737337112427,800.0 -200,300,100,800,900,0.07109089940786362,12.381159782409668,0.3241754174232483,800.0 -300,400,100,900,1000,0.06916239857673645,12.32018756866455,0.3376901149749756,800.0 -400,500,100,1000,1100,0.0810028612613678,11.741896629333496,0.38915595412254333,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index e78f9871ae6f151584bedd567e4d626c7c2fcc01..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06980103254318237,12.833678245544434,0.2880440354347229,800.0 -100,200,100,700,800,0.06914582848548889,12.34412956237793,0.3189157247543335,800.0 -200,300,100,800,900,0.07576127350330353,12.049708366394043,0.347758948802948,800.0 -300,400,100,900,1000,0.08593917638063431,11.535826683044434,0.37437546253204346,800.0 -400,500,100,1000,1100,0.10170240700244904,10.462043762207031,0.4152700901031494,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index f94a6ec18957021a3193d0d4924e48b60a2f8e15..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0565909668803215,14.000258445739746,0.2582165598869324,800.0 -100,200,100,700,800,0.06234465539455414,12.803106307983398,0.2871440649032593,800.0 -200,300,100,800,900,0.06456020474433899,12.934589385986328,0.2945743501186371,800.0 -300,400,100,900,1000,0.06268122792243958,12.847476959228516,0.3007753789424896,800.0 -400,500,100,1000,1100,0.07404273748397827,12.096294403076172,0.3447920083999634,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 94047ba2d5fa95e0ce064ebb3fe2f272785adeab..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05337296053767204,14.168307304382324,0.25284817814826965,800.0 -100,200,100,700,800,0.0647801011800766,12.585844993591309,0.30778419971466064,800.0 -200,300,100,800,900,0.07011856883764267,12.464638710021973,0.31545940041542053,800.0 -300,400,100,900,1000,0.07004537433385849,12.481287956237793,0.316457062959671,800.0 -400,500,100,1000,1100,0.08161136507987976,11.763980865478516,0.3621005117893219,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index 7cbc1c694a9faee6a2944c2bbba32530cfc15da9..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0595819391310215,13.977046012878418,0.3071853220462799,800.0 -100,200,100,700,800,0.05211150273680687,14.698359489440918,0.35139626264572144,800.0 -200,300,100,800,900,0.06747105717658997,13.296103477478027,0.3377290964126587,800.0 -300,400,100,900,1000,0.07107427716255188,12.436001777648926,0.4019967019557953,800.0 -400,500,100,1000,1100,0.049223147332668304,14.797995567321777,0.3630022406578064,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index a0692b1b399f06f71489f9e27bd142da5281ad7a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03298167884349823,19.484088897705078,0.20927543938159943,800.0 -100,200,100,700,800,0.07488890737295151,14.623353004455566,0.35173943638801575,800.0 -200,300,100,800,900,0.07676726579666138,14.997396469116211,0.34078431129455566,800.0 -300,400,100,900,1000,0.09353947639465332,11.484543800354004,0.4247261881828308,800.0 -400,500,100,1000,1100,0.07002867013216019,15.120462417602539,0.367643803358078,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index ff94efe5f6b486afb8a26cc1d3dbd1b413a17fbe..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03457418829202652,19.76328468322754,0.2005949765443802,800.0 -100,200,100,700,800,0.047808073461055756,17.28430938720703,0.2881235182285309,800.0 -200,300,100,800,900,0.050889622420072556,17.68299102783203,0.2729721963405609,800.0 -300,400,100,900,1000,0.07224664092063904,13.520915985107422,0.37655338644981384,800.0 -400,500,100,1000,1100,0.04121387004852295,17.60944175720215,0.3083353340625763,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index fb5a32b31eb0e5bda565ee907d3d9ab61060af89..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.030855856835842133,20.033714294433594,0.18423126637935638,800.0 -100,200,100,700,800,0.0556265152990818,16.553733825683594,0.2724183201789856,800.0 -200,300,100,800,900,0.04620935395359993,18.902416229248047,0.2666822373867035,800.0 -300,400,100,900,1000,0.06695535778999329,13.679944038391113,0.3559513986110687,800.0 -400,500,100,1000,1100,0.03810720518231392,17.82744598388672,0.27038800716400146,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 2b31f642ac54f9574d10f92ffe859ef8d3d3d76b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03389710560441017,19.683473587036133,0.201130211353302,800.0 -100,200,100,700,800,0.048803914338350296,17.016389846801758,0.2990020513534546,800.0 -200,300,100,800,900,0.05894763767719269,16.78786849975586,0.3154866397380829,800.0 -300,400,100,900,1000,0.07504153996706009,13.030791282653809,0.38903138041496277,800.0 -400,500,100,1000,1100,0.04184283688664436,17.08903694152832,0.3263438045978546,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index f7e24c0077e41bacb0639f4397fad97c0d110263..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02771022357046604,20.353904724121094,0.1605110615491867,800.0 -100,200,100,700,800,0.03114093281328678,18.772663116455078,0.20460379123687744,800.0 -200,300,100,800,900,0.04325484856963158,18.21112632751465,0.2430817186832428,800.0 -300,400,100,900,1000,0.06175437942147255,14.163307189941406,0.31930533051490784,800.0 -400,500,100,1000,1100,0.029290243983268738,19.059110641479492,0.23191988468170166,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 8f2257326812dd65fe4b6bcf906f2355cf2e1de0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025742752477526665,20.444541931152344,0.15347495675086975,800.0 -100,200,100,700,800,0.03508076071739197,17.52053451538086,0.2129494696855545,800.0 -200,300,100,800,900,0.040382348001003265,18.345497131347656,0.240972101688385,800.0 -300,400,100,900,1000,0.05818770453333855,14.330192565917969,0.3039705157279968,800.0 -400,500,100,1000,1100,0.0469200424849987,16.464088439941406,0.2846752405166626,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 06d8f2d7d2cf1587ecc03bf9eb6f382ba24604ed..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.026437722146511078,20.26837921142578,0.1607416570186615,800.0 -100,200,100,700,800,0.04232320189476013,17.440723419189453,0.23518553376197815,800.0 -200,300,100,800,900,0.05322279781103134,17.595478057861328,0.2817003130912781,800.0 -300,400,100,900,1000,0.0667048916220665,13.867680549621582,0.3397998511791229,800.0 -400,500,100,1000,1100,0.03546588495373726,18.173843383789062,0.27996206283569336,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index f44dc7a01fb4f318804ecce16d2d19b67b0f94b7..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025496507063508034,20.356264114379883,0.15741178393363953,800.0 -100,200,100,700,800,0.05755013972520828,16.680198669433594,0.2575400173664093,800.0 -200,300,100,800,900,0.04877137020230293,18.252368927001953,0.243723526597023,800.0 -300,400,100,900,1000,0.06076384708285332,14.109650611877441,0.29713183641433716,800.0 -400,500,100,1000,1100,0.04559295251965523,16.657651901245117,0.2855566143989563,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index 4b4021563f033659536da88a1ccf62f282faf783..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.036411136388778687,19.504281997680664,0.18136948347091675,800.0 -100,200,100,700,800,0.05701979249715805,16.561864852905273,0.24428808689117432,800.0 -200,300,100,800,900,0.05416395142674446,17.92426300048828,0.2578650712966919,800.0 -300,400,100,900,1000,0.06560200452804565,13.918689727783203,0.31258174777030945,800.0 -400,500,100,1000,1100,0.0512520931661129,17.083711624145508,0.29583823680877686,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 6540c05977fc762e238b0a36c90f23f905a56c5d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025536565110087395,19.885339736938477,0.1566312462091446,800.0 -100,200,100,700,800,0.049311310052871704,16.738481521606445,0.2332955151796341,800.0 -200,300,100,800,900,0.08651790022850037,14.691498756408691,0.2881164252758026,800.0 -300,400,100,900,1000,0.09129604697227478,12.536567687988281,0.31388428807258606,800.0 -400,500,100,1000,1100,0.08872195333242416,14.224748611450195,0.30216482281684875,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index 61d2a40a60088ed1811b5a82553be07655b00195..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0315806046128273,20.052783966064453,0.18573753535747528,800.0 -100,200,100,700,800,0.040987685322761536,17.95606803894043,0.2399943321943283,800.0 -200,300,100,800,900,0.04182804748415947,19.23689842224121,0.23884785175323486,800.0 -300,400,100,900,1000,0.059280719608068466,14.789297103881836,0.2981416583061218,800.0 -400,500,100,1000,1100,0.041447628289461136,17.9045352935791,0.26767590641975403,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index b38c017f51276b03205b58d54472f837b014ef5b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025667281821370125,20.998212814331055,0.14415240287780762,800.0 -100,200,100,700,800,0.03320636600255966,18.81351661682129,0.19365496933460236,800.0 -200,300,100,800,900,0.043277401477098465,19.246408462524414,0.2304617166519165,800.0 -300,400,100,900,1000,0.050798553973436356,15.437185287475586,0.26417404413223267,800.0 -400,500,100,1000,1100,0.03710523992776871,18.291786193847656,0.24844589829444885,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index d30c1d989e4f87fcd69b061a5bc8bf3e87a89c5c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.029333043843507767,19.72930145263672,0.15682512521743774,800.0 -100,200,100,700,800,0.050253067165613174,16.450895309448242,0.23974348604679108,800.0 -200,300,100,800,900,0.05453706905245781,17.998249053955078,0.2562529444694519,800.0 -300,400,100,900,1000,0.07603342831134796,13.7474946975708,0.2971600294113159,800.0 -400,500,100,1000,1100,0.04818221554160118,17.346128463745117,0.2538471519947052,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 2b81f19862d72555c25e59dfec95f659a4ab4f48..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02736622281372547,20.841440200805664,0.1578013300895691,800.0 -100,200,100,700,800,0.05652492120862007,16.485580444335938,0.268746554851532,800.0 -200,300,100,800,900,0.05829903110861778,17.238191604614258,0.2833211421966553,800.0 -300,400,100,900,1000,0.06662929058074951,14.275581359863281,0.312130868434906,800.0 -400,500,100,1000,1100,0.04551764577627182,16.349552154541016,0.2924472391605377,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 0dbc2661c725e075c57ac8b4bcc374d5f8a67e91..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02179647609591484,21.013629913330078,0.1515733152627945,800.0 -100,200,100,700,800,0.029498102143406868,18.3458251953125,0.19541211426258087,800.0 -200,300,100,800,900,0.03190116584300995,19.214876174926758,0.20869146287441254,800.0 -300,400,100,900,1000,0.04469861835241318,15.843648910522461,0.21773988008499146,800.0 -400,500,100,1000,1100,0.036886297166347504,17.701345443725586,0.22379209101200104,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 5072c8d3fcb4e3944023f66f628d6ce0b581143c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.028334813192486763,20.371967315673828,0.16474394500255585,800.0 -100,200,100,700,800,0.06444521248340607,16.12744903564453,0.271580308675766,800.0 -200,300,100,800,900,0.05265073850750923,18.129255294799805,0.2717227339744568,800.0 -300,400,100,900,1000,0.05958159267902374,14.368860244750977,0.282820463180542,800.0 -400,500,100,1000,1100,0.05764060840010643,16.090497970581055,0.27224767208099365,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 25e1e4fb1564382dd7380122c7185d7ecd5d7772..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02414107881486416,21.20865249633789,0.15156210958957672,800.0 -100,200,100,700,800,0.05886954441666603,16.880159378051758,0.25297310948371887,800.0 -200,300,100,800,900,0.07205526530742645,17.553993225097656,0.29177403450012207,800.0 -300,400,100,900,1000,0.09192748367786407,13.578813552856445,0.3569633364677429,800.0 -400,500,100,1000,1100,0.08682157844305038,15.62374496459961,0.33470696210861206,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index cff56ec3e7516361dcd742e0fc22f5f564ef4e01..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03344518318772316,20.098159790039062,0.1717255711555481,800.0 -100,200,100,700,800,0.044739607721567154,17.276020050048828,0.24522292613983154,800.0 -200,300,100,800,900,0.04570399597287178,18.699682235717773,0.2589333951473236,800.0 -300,400,100,900,1000,0.08021324872970581,13.279784202575684,0.3522612750530243,800.0 -400,500,100,1000,1100,0.051871832460165024,17.637977600097656,0.2812711000442505,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 1dd3d953b364b8b3d4e9311aa40e123efb2e0b1c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05070306733250618,17.889474868774414,0.256004273891449,800.0 -100,200,100,700,800,0.06780268996953964,14.776022911071777,0.3344288170337677,800.0 -200,300,100,800,900,0.07670124620199203,15.385300636291504,0.32320448756217957,800.0 -300,400,100,900,1000,0.09557755291461945,11.930201530456543,0.4115227460861206,800.0 -400,500,100,1000,1100,0.1714160293340683,11.072236061096191,0.41516873240470886,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 6efdbe1325b8aa227fbede1d5e36bb32ccf4ef3b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07350466400384903,13.608785629272461,0.3550418019294739,800.0 -100,200,100,700,800,0.08502659946680069,12.651140213012695,0.4216804504394531,800.0 -200,300,100,800,900,0.10062836855649948,12.029831886291504,0.44783729314804077,800.0 -300,400,100,900,1000,0.09573697298765182,11.094364166259766,0.5303404331207275,800.0 -400,500,100,1000,1100,0.13560712337493896,10.247191429138184,0.5950062274932861,800.0 diff --git a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index ab180f5d5ef5c43b1430480f915b6107cd0929af..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_anchorclean_noncausal_a2d4r2_route_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192412-rneyh4x0/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06191617622971535,15.532018661499023,0.32631051540374756,800.0 -100,200,100,700,800,0.10068749636411667,11.513010025024414,0.4595262408256531,800.0 -200,300,100,800,900,0.10488159209489822,10.859790802001953,0.45879971981048584,800.0 -300,400,100,900,1000,0.12318231910467148,9.654779434204102,0.536268949508667,800.0 -400,500,100,1000,1100,0.10296685993671417,10.995999336242676,0.4752253592014313,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 6bfad204234ad84e30f69f4437bdd48d78e90291..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.09916018694639206,11.130820274353027,0.39710304141044617,800.0 -100,200,100,700,800,0.11695688217878342,9.566265106201172,0.5011105537414551,800.0 -200,300,100,800,900,0.15700799226760864,8.404092788696289,0.5823686122894287,800.0 -300,400,100,900,1000,0.13034145534038544,9.252965927124023,0.5381281971931458,800.0 -400,500,100,1000,1100,0.1320589780807495,9.344514846801758,0.5381790399551392,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index ec8090e77362b399ffa2d6c6557dd5ce08fe118e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07996285706758499,12.076528549194336,0.3373993933200836,800.0 -100,200,100,700,800,0.12075352668762207,9.542525291442871,0.4611355662345886,800.0 -200,300,100,800,900,0.1384482979774475,9.126065254211426,0.491639107465744,800.0 -300,400,100,900,1000,0.11553388833999634,9.77892780303955,0.4398800730705261,800.0 -400,500,100,1000,1100,0.13375718891620636,9.493799209594727,0.49055084586143494,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index a51cf8e8f82bd00be9e3e9495678df608e3180a0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07604966312646866,12.365468978881836,0.3209730386734009,800.0 -100,200,100,700,800,0.09144487231969833,10.869845390319824,0.3894427418708801,800.0 -200,300,100,800,900,0.09682945162057877,10.635035514831543,0.40785446763038635,800.0 -300,400,100,900,1000,0.0853242501616478,11.536821365356445,0.37019166350364685,800.0 -400,500,100,1000,1100,0.08978407084941864,11.136824607849121,0.41134780645370483,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index b2d56019e1ac23e1764862ae4971fdf9cefebf95..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.065084308385849,13.051470756530762,0.3065177798271179,800.0 -100,200,100,700,800,0.08221898227930069,11.355535507202148,0.37691619992256165,800.0 -200,300,100,800,900,0.08118075132369995,11.742218971252441,0.35331374406814575,800.0 -300,400,100,900,1000,0.08292558044195175,11.706655502319336,0.3536755442619324,800.0 -400,500,100,1000,1100,0.09258215129375458,11.024810791015625,0.3988085985183716,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 1a0b0d9579ce4e5edfacf22b3c41dbe17edc65d2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06994319707155228,12.784317970275879,0.3196448087692261,800.0 -100,200,100,700,800,0.08118125051259995,11.610411643981934,0.35627442598342896,800.0 -200,300,100,800,900,0.09483057260513306,11.080421447753906,0.378732293844223,800.0 -300,400,100,900,1000,0.10156547278165817,10.473685264587402,0.3930116295814514,800.0 -400,500,100,1000,1100,0.1066589280962944,10.29081916809082,0.4194580018520355,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 2fb9e334a723cb93319df96a9a41be4b74638870..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0758124515414238,12.445165634155273,0.330237478017807,800.0 -100,200,100,700,800,0.11103661358356476,9.833064079284668,0.4274407923221588,800.0 -200,300,100,800,900,0.12182185798883438,9.714326858520508,0.42325347661972046,800.0 -300,400,100,900,1000,0.15083055198192596,8.861239433288574,0.46643829345703125,800.0 -400,500,100,1000,1100,0.1430736482143402,8.970564842224121,0.49306395649909973,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index c53e709a8076f6e7ea41be8ba97afa3c64b6c145..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06537093967199326,13.008379936218262,0.3027077615261078,800.0 -100,200,100,700,800,0.0862167477607727,11.330513954162598,0.37251463532447815,800.0 -200,300,100,800,900,0.10880538076162338,10.673624038696289,0.38427260518074036,800.0 -300,400,100,900,1000,0.13447223603725433,9.909564018249512,0.3943711817264557,800.0 -400,500,100,1000,1100,0.15632747113704681,8.898813247680664,0.4667634665966034,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 9382978d8cc673ef7930a7ab87e821b65a2fb832..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.10699569433927536,11.342495918273926,0.3665500581264496,800.0 -100,200,100,700,800,0.1713337004184723,8.10660457611084,0.5098422169685364,800.0 -200,300,100,800,900,0.18415524065494537,7.867537021636963,0.5371197462081909,800.0 -300,400,100,900,1000,0.2100457400083542,7.4717841148376465,0.5628650188446045,800.0 -400,500,100,1000,1100,0.22128981351852417,7.02705192565918,0.6208775043487549,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index dd9edabe3c28105250f7b964bc7cf6024f79da99..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06512743979692459,13.044225692749023,0.30791065096855164,800.0 -100,200,100,700,800,0.10054199397563934,10.72892951965332,0.39151108264923096,800.0 -200,300,100,800,900,0.15377329289913177,9.150006294250488,0.4250929355621338,800.0 -300,400,100,900,1000,0.22229309380054474,7.81513786315918,0.4923965334892273,800.0 -400,500,100,1000,1100,0.22962802648544312,7.205636024475098,0.5590951442718506,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 8a2216eb23bb98c19a036887f26c33ab6564ca08..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.09177803248167038,11.991604804992676,0.3468701243400574,800.0 -100,200,100,700,800,0.1827676147222519,8.077796936035156,0.4822515845298767,800.0 -200,300,100,800,900,0.25628843903541565,6.233086585998535,0.5279352068901062,800.0 -300,400,100,900,1000,0.2760530412197113,5.982347011566162,0.5284307599067688,800.0 -400,500,100,1000,1100,0.28539323806762695,5.769464492797852,0.5985733270645142,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index a2cf5682a4ccb3b6a12c9004903f25599a25d491..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07265175133943558,12.839853286743164,0.31920862197875977,800.0 -100,200,100,700,800,0.09797007590532303,10.798120498657227,0.3939485251903534,800.0 -200,300,100,800,900,0.09550946205854416,10.906440734863281,0.3849441409111023,800.0 -300,400,100,900,1000,0.1645013839006424,8.882270812988281,0.42377060651779175,800.0 -400,500,100,1000,1100,0.1701011061668396,8.966602325439453,0.45859450101852417,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index 43b5a448f34c9e9f80611310bbd37e4f7f4a1300..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06964889913797379,12.81812858581543,0.3111996352672577,800.0 -100,200,100,700,800,0.11895541846752167,9.896576881408691,0.41394928097724915,800.0 -200,300,100,800,900,0.20172053575515747,7.863762378692627,0.4599761962890625,800.0 -300,400,100,900,1000,0.23882673680782318,7.149023532867432,0.5304481387138367,800.0 -400,500,100,1000,1100,0.20577804744243622,7.450506687164307,0.5488693118095398,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index c326d7636f7b7ebe1409b7e9b1729c466e10b2c2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08944860100746155,11.87744140625,0.3367711007595062,800.0 -100,200,100,700,800,0.15420636534690857,8.633942604064941,0.4817330837249756,800.0 -200,300,100,800,900,0.19503775238990784,7.8537116050720215,0.5347532629966736,800.0 -300,400,100,900,1000,0.20889978110790253,7.554243087768555,0.5177484154701233,800.0 -400,500,100,1000,1100,0.2360023856163025,6.544476509094238,0.5902954339981079,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 5fbe8a8a1c39ee8b26158fedfdf7ced3a7fb482a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06378071010112762,12.898210525512695,0.3179067075252533,800.0 -100,200,100,700,800,0.1017589122056961,10.51761245727539,0.3991122543811798,800.0 -200,300,100,800,900,0.15403366088867188,8.956668853759766,0.46200454235076904,800.0 -300,400,100,900,1000,0.19709977507591248,7.9705586433410645,0.47065383195877075,800.0 -400,500,100,1000,1100,0.22403191030025482,7.622913837432861,0.5136918425559998,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index 8427ab5523c74f2e632324f6bf702ca708633935..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12002597749233246,11.329145431518555,0.3796159625053406,800.0 -100,200,100,700,800,0.23740024864673615,7.169523239135742,0.5568605065345764,800.0 -200,300,100,800,900,0.2639369070529938,6.6201372146606445,0.5206961035728455,800.0 -300,400,100,900,1000,0.29212304949760437,5.492745399475098,0.6169043183326721,800.0 -400,500,100,1000,1100,0.33833634853363037,4.840957164764404,0.6235620379447937,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index fd5c489ba53eed65cd9df66f15e7679596a826f8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07243677973747253,16.21759033203125,0.2715952694416046,800.0 -100,200,100,700,800,0.18451984226703644,9.904926300048828,0.43301764130592346,800.0 -200,300,100,800,900,0.2245563119649887,11.35063362121582,0.4235581159591675,800.0 -300,400,100,900,1000,0.19537967443466187,9.132794380187988,0.48037710785865784,800.0 -400,500,100,1000,1100,0.2258927971124649,8.90002727508545,0.48758843541145325,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index df90671f759d26eeba555941b8852c09ca898e3d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07819069176912308,16.440536499023438,0.27301260828971863,800.0 -100,200,100,700,800,0.2973203957080841,6.6711812019348145,0.5243391990661621,800.0 -200,300,100,800,900,0.33593982458114624,5.274209499359131,0.5339179039001465,800.0 -300,400,100,900,1000,0.3666591942310333,4.878693103790283,0.5886023640632629,800.0 -400,500,100,1000,1100,0.4658375680446625,3.5448801517486572,0.632232666015625,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index ae2d2a072b10557a9d7ba5bb3485e55a36e53f37..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05709133669734001,18.281801223754883,0.2502315640449524,800.0 -100,200,100,700,800,0.12045673280954361,13.937300682067871,0.33855685591697693,800.0 -200,300,100,800,900,0.117275170981884,14.631621360778809,0.36156749725341797,800.0 -300,400,100,900,1000,0.1470295488834381,9.77096176147461,0.451699823141098,800.0 -400,500,100,1000,1100,0.1941142976284027,11.268633842468262,0.45253631472587585,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 68a01f43505673eb22f7bca72e09cc824fcbde4e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07526741176843643,17.42080307006836,0.2683486342430115,800.0 -100,200,100,700,800,0.17283210158348083,11.201020240783691,0.41833701729774475,800.0 -200,300,100,800,900,0.1481478214263916,13.295750617980957,0.39461031556129456,800.0 -300,400,100,900,1000,0.1468915045261383,9.96481990814209,0.4559366703033447,800.0 -400,500,100,1000,1100,0.1873568296432495,10.833941459655762,0.465509831905365,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 5b612836c9236e613e913f4231e8b616e66cb4e9..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04850547015666962,18.851411819458008,0.2272331565618515,800.0 -100,200,100,700,800,0.06705111265182495,15.26913070678711,0.302547425031662,800.0 -200,300,100,800,900,0.05760632082819939,17.334047317504883,0.29314205050468445,800.0 -300,400,100,900,1000,0.08143764734268188,12.88891887664795,0.3772006928920746,800.0 -400,500,100,1000,1100,0.0792287290096283,14.593842506408691,0.33466485142707825,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 95e848e26d48a97dd6c3aa1502b5c1531bf4e56f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.045679181814193726,19.112333297729492,0.21882936358451843,800.0 -100,200,100,700,800,0.07529930770397186,14.577890396118164,0.31246259808540344,800.0 -200,300,100,800,900,0.14301863312721252,13.134121894836426,0.37338295578956604,800.0 -300,400,100,900,1000,0.18789078295230865,8.318058013916016,0.4821917712688446,800.0 -400,500,100,1000,1100,0.25416579842567444,9.243608474731445,0.4825778305530548,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index ed9b47efee31c89d7b982324704e1f3e54b1d74a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.058957405388355255,18.181110382080078,0.24730294942855835,800.0 -100,200,100,700,800,0.13120102882385254,13.152020454406738,0.3689264953136444,800.0 -200,300,100,800,900,0.1572161465883255,13.006508827209473,0.3936946392059326,800.0 -300,400,100,900,1000,0.15029114484786987,10.333911895751953,0.442178875207901,800.0 -400,500,100,1000,1100,0.18656879663467407,10.65812873840332,0.4577710032463074,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 29bdae22afdab6cec356fb520485d5b6b699c25b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06487486511468887,18.13372230529785,0.24573716521263123,800.0 -100,200,100,700,800,0.11761467903852463,13.344557762145996,0.35964295268058777,800.0 -200,300,100,800,900,0.10310287773609161,14.887663841247559,0.36251693964004517,800.0 -300,400,100,900,1000,0.1075775995850563,11.499134063720703,0.43049073219299316,800.0 -400,500,100,1000,1100,0.15343433618545532,11.42818832397461,0.4280698001384735,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index f7162ac38e5fd5357032927e75e2e7e28cbcb708..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04465317353606224,19.008407592773438,0.21295207738876343,800.0 -100,200,100,700,800,0.10480190068483353,13.482619285583496,0.341155081987381,800.0 -200,300,100,800,900,0.13500624895095825,13.852980613708496,0.37104740738868713,800.0 -300,400,100,900,1000,0.10193055868148804,11.874672889709473,0.4057210087776184,800.0 -400,500,100,1000,1100,0.16143698990345,11.969481468200684,0.4132092297077179,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index bf88b1d5b1cc468fe2be2930fa260a1c5a8d4065..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.045760005712509155,18.875770568847656,0.22084078192710876,800.0 -100,200,100,700,800,0.1314617097377777,12.927082061767578,0.36748412251472473,800.0 -200,300,100,800,900,0.1770194172859192,11.997867584228516,0.4101102352142334,800.0 -300,400,100,900,1000,0.19429703056812286,8.117560386657715,0.4929509460926056,800.0 -400,500,100,1000,1100,0.2791578769683838,8.873279571533203,0.4823230803012848,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index 194b5b85b330074dfad3dee3ef7542d33d250839..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05331391468644142,18.0147762298584,0.23932823538780212,800.0 -100,200,100,700,800,0.1034725159406662,14.041906356811523,0.32517755031585693,800.0 -200,300,100,800,900,0.08397199958562851,15.74539566040039,0.33538228273391724,800.0 -300,400,100,900,1000,0.0734647735953331,12.76567268371582,0.37564927339553833,800.0 -400,500,100,1000,1100,0.08099645376205444,13.894255638122559,0.375565767288208,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index eaa98d7e5395233718434312f22314c7bae2fe83..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04161849617958069,19.48943519592285,0.1999492645263672,800.0 -100,200,100,700,800,0.18711014091968536,11.868228912353516,0.37467747926712036,800.0 -200,300,100,800,900,0.2633311152458191,11.601203918457031,0.435550332069397,800.0 -300,400,100,900,1000,0.25659632682800293,8.852799415588379,0.45802414417266846,800.0 -400,500,100,1000,1100,0.3414517641067505,8.74924087524414,0.4927671551704407,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 8b3ce3f78b438fef9916a5fabedf18c5318667c5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05507046356797218,18.69900131225586,0.24234743416309357,800.0 -100,200,100,700,800,0.08676453679800034,14.824301719665527,0.33592405915260315,800.0 -200,300,100,800,900,0.089546337723732,15.155591011047363,0.34316933155059814,800.0 -300,400,100,900,1000,0.08365222811698914,12.736736297607422,0.3852022588253021,800.0 -400,500,100,1000,1100,0.11383367329835892,12.383835792541504,0.4069167375564575,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 41d281003283415ca1fb915aee4b16e572672bbc..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05378216877579689,17.943208694458008,0.23068538308143616,800.0 -100,200,100,700,800,0.10898252576589584,12.908111572265625,0.35172417759895325,800.0 -200,300,100,800,900,0.13582895696163177,13.966197967529297,0.3677370548248291,800.0 -300,400,100,900,1000,0.12249278277158737,11.262565612792969,0.4276915490627289,800.0 -400,500,100,1000,1100,0.20249736309051514,10.33936882019043,0.4339638650417328,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index f879f761f86e509ceca5e46f219cfd04ba3c49cb..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04794800654053688,18.575986862182617,0.22708486020565033,800.0 -100,200,100,700,800,0.13439399003982544,12.942028999328613,0.3654666841030121,800.0 -200,300,100,800,900,0.16886906325817108,12.815217018127441,0.3769916892051697,800.0 -300,400,100,900,1000,0.14126130938529968,10.10905933380127,0.4615817964076996,800.0 -400,500,100,1000,1100,0.1667911410331726,11.319565773010254,0.4002702832221985,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index d5341cd51b06f1fc46f42e8995be1a904a13825f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07303345948457718,17.576873779296875,0.25926074385643005,800.0 -100,200,100,700,800,0.19638104736804962,10.293671607971191,0.43231865763664246,800.0 -200,300,100,800,900,0.19748584926128387,12.129781723022461,0.4430329501628876,800.0 -300,400,100,900,1000,0.21924664080142975,8.527020454406738,0.49403446912765503,800.0 -400,500,100,1000,1100,0.30535051226615906,8.261244773864746,0.5293222665786743,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 2f722f6f76739361798f360575d03cb544bf566e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06794432550668716,18.594436645507812,0.23109447956085205,800.0 -100,200,100,700,800,0.21349261701107025,11.476329803466797,0.4305765628814697,800.0 -200,300,100,800,900,0.27062615752220154,9.811241149902344,0.44287315011024475,800.0 -300,400,100,900,1000,0.2649693191051483,7.100702285766602,0.520061731338501,800.0 -400,500,100,1000,1100,0.387721449136734,6.197927951812744,0.5427039861679077,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 2017c15a0cf0fff4b380da84e0e692e156e02124..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04942026734352112,18.875524520874023,0.19586578011512756,800.0 -100,200,100,700,800,0.2640610933303833,9.654930114746094,0.43290725350379944,800.0 -200,300,100,800,900,0.2643427848815918,11.236988067626953,0.43312209844589233,800.0 -300,400,100,900,1000,0.2521970570087433,8.460708618164062,0.4807858169078827,800.0 -400,500,100,1000,1100,0.32791537046432495,7.547647476196289,0.48594582080841064,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 7bda67de2f4ea459f9cccdea50b1b306585a51e6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.054791565984487534,18.60540199279785,0.24284915626049042,800.0 -100,200,100,700,800,0.07952892780303955,14.219416618347168,0.3287675976753235,800.0 -200,300,100,800,900,0.0838644951581955,16.565671920776367,0.33700135350227356,800.0 -300,400,100,900,1000,0.10098277777433395,11.77396011352539,0.40974777936935425,800.0 -400,500,100,1000,1100,0.09818706661462784,11.789966583251953,0.38952919840812683,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 853900d00823eca8cda024fa890519ac30212cab..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05795349180698395,18.30255889892578,0.23208509385585785,800.0 -100,200,100,700,800,0.13707874715328217,12.927743911743164,0.3716380298137665,800.0 -200,300,100,800,900,0.1594209522008896,14.968881607055664,0.3728829622268677,800.0 -300,400,100,900,1000,0.173130601644516,10.438394546508789,0.44330117106437683,800.0 -400,500,100,1000,1100,0.2765843868255615,10.556353569030762,0.452711284160614,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index bdee7928c9eebe4255ae5d5559965349272f16e6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12118896842002869,15.932927131652832,0.30224883556365967,800.0 -100,200,100,700,800,0.2134353667497635,9.659161567687988,0.4774169921875,800.0 -200,300,100,800,900,0.16573770344257355,12.230481147766113,0.4276477098464966,800.0 -300,400,100,900,1000,0.13384242355823517,10.356162071228027,0.4343750476837158,800.0 -400,500,100,1000,1100,0.2184433937072754,9.57994556427002,0.49065887928009033,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index 3a814c011a8e627925f24e4b3c72b8f1250315db..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07376087456941605,17.778886795043945,0.2636793255805969,800.0 -100,200,100,700,800,0.24373891949653625,9.48464584350586,0.48810064792633057,800.0 -200,300,100,800,900,0.2695155143737793,10.069662094116211,0.48554378747940063,800.0 -300,400,100,900,1000,0.258342981338501,7.624960899353027,0.5085524320602417,800.0 -400,500,100,1000,1100,0.3156861960887909,8.579822540283203,0.4964810609817505,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 3f4d8c86811ebc6b7ea9834bbe126f049b5fb9f4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04166504368185997,19.55937957763672,0.2018101066350937,800.0 -100,200,100,700,800,0.0989910215139389,15.243603706359863,0.3113028407096863,800.0 -200,300,100,800,900,0.10889941453933716,16.08378791809082,0.32984793186187744,800.0 -300,400,100,900,1000,0.11462872475385666,11.958256721496582,0.39189499616622925,800.0 -400,500,100,1000,1100,0.13769173622131348,14.236289024353027,0.3601906895637512,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index ebe8f3754c5eb4913cc42304d54a9e8e2b139166..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04787260666489601,19.410682678222656,0.20794180035591125,800.0 -100,200,100,700,800,0.10700536519289017,13.478046417236328,0.35218992829322815,800.0 -200,300,100,800,900,0.07381691783666611,17.004241943359375,0.31669312715530396,800.0 -300,400,100,900,1000,0.08379285782575607,12.687128067016602,0.3661843240261078,800.0 -400,500,100,1000,1100,0.09047975391149521,15.591073036193848,0.3404996991157532,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index a62e48e2093d3253fbb2961f71d2dcb75fc46710..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08616583049297333,17.581005096435547,0.2566639482975006,800.0 -100,200,100,700,800,0.20904015004634857,10.319645881652832,0.4255390167236328,800.0 -200,300,100,800,900,0.20665313303470612,12.556397438049316,0.4261779487133026,800.0 -300,400,100,900,1000,0.19251832365989685,9.796375274658203,0.4972895085811615,800.0 -400,500,100,1000,1100,0.2501325011253357,9.843011856079102,0.45158395171165466,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index f2961a65a85a26ed7849a0ede55dcb69989cad73..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06527653336524963,17.692642211914062,0.2427273988723755,800.0 -100,200,100,700,800,0.14962896704673767,11.620125770568848,0.3856595754623413,800.0 -200,300,100,800,900,0.16118545830249786,13.145500183105469,0.3832263946533203,800.0 -300,400,100,900,1000,0.1168292686343193,11.1198148727417,0.4090065062046051,800.0 -400,500,100,1000,1100,0.17981897294521332,10.49620246887207,0.444885790348053,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 0b1d8ea1d6fea5ceee18f88e6d1ff86af660c1de..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06897089630365372,18.097532272338867,0.23961776494979858,800.0 -100,200,100,700,800,0.1503850519657135,12.177825927734375,0.3739897608757019,800.0 -200,300,100,800,900,0.14241203665733337,14.08682632446289,0.37858039140701294,800.0 -300,400,100,900,1000,0.13378503918647766,10.554328918457031,0.430012583732605,800.0 -400,500,100,1000,1100,0.18851202726364136,10.344286918640137,0.4117194712162018,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index dd62bcc5bf13b700dad0a812c2397931688fbfa0..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0615512989461422,18.470909118652344,0.2220134139060974,800.0 -100,200,100,700,800,0.18742473423480988,11.113094329833984,0.40758001804351807,800.0 -200,300,100,800,900,0.18431468307971954,12.979462623596191,0.4093693792819977,800.0 -300,400,100,900,1000,0.1417403668165207,11.001697540283203,0.43504294753074646,800.0 -400,500,100,1000,1100,0.1947994828224182,10.903644561767578,0.44587084650993347,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index da179513ca95164d636ee3a1505e5dd71f453f36..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.057084862142801285,17.946130752563477,0.23498623073101044,800.0 -100,200,100,700,800,0.14532601833343506,13.284667015075684,0.3569188714027405,800.0 -200,300,100,800,900,0.1549639254808426,14.600983619689941,0.3566988706588745,800.0 -300,400,100,900,1000,0.13769201934337616,10.96487808227539,0.4253823161125183,800.0 -400,500,100,1000,1100,0.18425308167934418,12.385558128356934,0.3839145004749298,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 436224fcfed9806c1e9eb55426da97732f6b3056..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08132030814886093,18.0466365814209,0.23670648038387299,800.0 -100,200,100,700,800,0.271481454372406,9.577834129333496,0.4573887586593628,800.0 -200,300,100,800,900,0.29450345039367676,11.044254302978516,0.4462811350822449,800.0 -300,400,100,900,1000,0.2816174626350403,7.463213920593262,0.5167691707611084,800.0 -400,500,100,1000,1100,0.36928215622901917,7.161813735961914,0.5230341553688049,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index a6c7e9f750d0a0c8db3bca52f621effa2a397e13..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07107288390398026,18.222789764404297,0.25406157970428467,800.0 -100,200,100,700,800,0.1425357609987259,13.077061653137207,0.3844691812992096,800.0 -200,300,100,800,900,0.1415289044380188,14.187338829040527,0.3813498020172119,800.0 -300,400,100,900,1000,0.12145106494426727,11.090134620666504,0.43445292115211487,800.0 -400,500,100,1000,1100,0.1333943009376526,12.081716537475586,0.41669589281082153,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index d8ed284fbbff168e5531258f13051dad86e4c342..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.07639294862747192,16.920019149780273,0.26039591431617737,800.0 -100,200,100,700,800,0.2026061862707138,9.957755088806152,0.4584100842475891,800.0 -200,300,100,800,900,0.20023055374622345,10.19502067565918,0.4610140323638916,800.0 -300,400,100,900,1000,0.18422412872314453,7.647907733917236,0.49772071838378906,800.0 -400,500,100,1000,1100,0.2595376670360565,6.4857330322265625,0.5211317539215088,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index 7950c4123f5db09c14a34bcfe022dc7c2db18693..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.10540880262851715,17.657655715942383,0.25972941517829895,800.0 -100,200,100,700,800,0.26232606172561646,9.856566429138184,0.43266531825065613,800.0 -200,300,100,800,900,0.31339454650878906,10.989925384521484,0.45741841197013855,800.0 -300,400,100,900,1000,0.30896589159965515,7.678021430969238,0.5144568681716919,800.0 -400,500,100,1000,1100,0.4156546890735626,7.428073883056641,0.5199502110481262,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index aba1898f7705c287d3140275e4f3030c596f46c4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.046741046011447906,19.390214920043945,0.2029205709695816,800.0 -100,200,100,700,800,0.14884749054908752,12.430472373962402,0.3808571696281433,800.0 -200,300,100,800,900,0.22054795920848846,12.0776948928833,0.4149623513221741,800.0 -300,400,100,900,1000,0.25322777032852173,8.577686309814453,0.45879432559013367,800.0 -400,500,100,1000,1100,0.31647488474845886,9.254781723022461,0.4570000171661377,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index b5560447429a1cb631a0a3b07b5255da26549a46..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0396239198744297,20.057594299316406,0.19528110325336456,800.0 -100,200,100,700,800,0.13801361620426178,13.882369041442871,0.3461547791957855,800.0 -200,300,100,800,900,0.12968531250953674,14.627602577209473,0.35362833738327026,800.0 -300,400,100,900,1000,0.13847395777702332,11.023517608642578,0.42940956354141235,800.0 -400,500,100,1000,1100,0.19046038389205933,11.862763404846191,0.4306744635105133,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index bffecf5ec27507280a37aa8c28c196f4f312348a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06542318314313889,18.349851608276367,0.23172520101070404,800.0 -100,200,100,700,800,0.11595003306865692,13.849248886108398,0.33186236023902893,800.0 -200,300,100,800,900,0.1640724241733551,13.181451797485352,0.38243043422698975,800.0 -300,400,100,900,1000,0.17244447767734528,9.721966743469238,0.4672190845012665,800.0 -400,500,100,1000,1100,0.23394975066184998,9.672555923461914,0.4536531865596771,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index e15c080c1a77af1303ce1c8359a9a89854cf465b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.1689700335264206,14.735254287719727,0.32092487812042236,800.0 -100,200,100,700,800,0.3130391836166382,8.803930282592773,0.47203734517097473,800.0 -200,300,100,800,900,0.30745115876197815,10.76585578918457,0.4665159285068512,800.0 -300,400,100,900,1000,0.3146374821662903,7.606332302093506,0.5470142364501953,800.0 -400,500,100,1000,1100,0.38025981187820435,8.075235366821289,0.5309199094772339,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index 495776387a60a2814c8d705c2236d470b7262c1e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.051032133400440216,19.03021240234375,0.21368062496185303,800.0 -100,200,100,700,800,0.11832935363054276,13.700409889221191,0.3562065064907074,800.0 -200,300,100,800,900,0.1465967297554016,14.829293251037598,0.3560546934604645,800.0 -300,400,100,900,1000,0.15922579169273376,10.629556655883789,0.42419230937957764,800.0 -400,500,100,1000,1100,0.2509492039680481,11.658411026000977,0.397975355386734,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index af565dd0d5160207c31476bff41331113d9f04c1..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.051279325038194656,19.004919052124023,0.21867533028125763,800.0 -100,200,100,700,800,0.1032533049583435,13.771475791931152,0.3560031056404114,800.0 -200,300,100,800,900,0.09949671477079391,14.789643287658691,0.36606934666633606,800.0 -300,400,100,900,1000,0.09277629852294922,12.085698127746582,0.4074211120605469,800.0 -400,500,100,1000,1100,0.12971478700637817,12.597463607788086,0.3959764540195465,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 6a5bbfc8aadb6422e4c13e74aa3828aeb7a373bf..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06559199094772339,18.017797470092773,0.2248487025499344,800.0 -100,200,100,700,800,0.21138572692871094,10.00480842590332,0.412923663854599,800.0 -200,300,100,800,900,0.19643883407115936,11.309859275817871,0.40910038352012634,800.0 -300,400,100,900,1000,0.160922572016716,9.643085479736328,0.44104477763175964,800.0 -400,500,100,1000,1100,0.26523900032043457,8.401888847351074,0.4592409133911133,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 37e0903a79672e3ec44bd03a92afc1dba70b5ed2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.054275032132864,18.167055130004883,0.22311076521873474,800.0 -100,200,100,700,800,0.1190994530916214,13.579154968261719,0.34902504086494446,800.0 -200,300,100,800,900,0.1610368937253952,10.547370910644531,0.4038190543651581,800.0 -300,400,100,900,1000,0.21901282668113708,8.236597061157227,0.5232927203178406,800.0 -400,500,100,1000,1100,0.3287922739982605,5.650357723236084,0.5573605298995972,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 450aa4cf4a8112474e91bad66dafee0cff787026..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08620914816856384,16.913681030273438,0.2722149193286896,800.0 -100,200,100,700,800,0.17724274098873138,11.108992576599121,0.41233283281326294,800.0 -200,300,100,800,900,0.143995463848114,13.393546104431152,0.39373940229415894,800.0 -300,400,100,900,1000,0.17771786451339722,9.953309059143066,0.460355669260025,800.0 -400,500,100,1000,1100,0.23981964588165283,9.926289558410645,0.4488258361816406,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 2d177d4878d486e8ad85b1a30129a256d0498e9d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0447663813829422,18.917930603027344,0.20770221948623657,800.0 -100,200,100,700,800,0.14158689975738525,13.12796401977539,0.35937464237213135,800.0 -200,300,100,800,900,0.15415826439857483,13.7755708694458,0.3713100850582123,800.0 -300,400,100,900,1000,0.1439480036497116,10.901330947875977,0.4273589253425598,800.0 -400,500,100,1000,1100,0.1947634518146515,11.281804084777832,0.42218101024627686,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 9bd77c6965cedb2acaaa52e5334e0c08062b95a6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12705062329769135,15.270642280578613,0.3165558874607086,800.0 -100,200,100,700,800,0.22163125872612,9.34744930267334,0.46299993991851807,800.0 -200,300,100,800,900,0.2404058426618576,10.142672538757324,0.43542560935020447,800.0 -300,400,100,900,1000,0.21657049655914307,8.306815147399902,0.4938114881515503,800.0 -400,500,100,1000,1100,0.32196176052093506,7.157313346862793,0.5159227252006531,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index 60d95ac0cd3bfedbaf227f3d014e3010b653530d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06944602727890015,18.344663619995117,0.2265375852584839,800.0 -100,200,100,700,800,0.2093111276626587,10.216194152832031,0.43340441584587097,800.0 -200,300,100,800,900,0.2061234712600708,11.09404182434082,0.43443259596824646,800.0 -300,400,100,900,1000,0.18958573043346405,9.388232231140137,0.44633471965789795,800.0 -400,500,100,1000,1100,0.2834295332431793,8.695345878601074,0.49879273772239685,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index a3c139369d4aff947eae6575aca81ed0f516b23e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.09215760976076126,17.179601669311523,0.2605637311935425,800.0 -100,200,100,700,800,0.1900012046098709,10.911874771118164,0.4086286425590515,800.0 -200,300,100,800,900,0.19089774787425995,11.337935447692871,0.43344351649284363,800.0 -300,400,100,900,1000,0.17108066380023956,9.492403984069824,0.4706188440322876,800.0 -400,500,100,1000,1100,0.21624240279197693,10.208394050598145,0.4381832182407379,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index df88be316b9302695a42aac5c0d1de477514b0ba..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.12322387844324112,15.350765228271484,0.31733158230781555,800.0 -100,200,100,700,800,0.20600482821464539,10.19345474243164,0.44197821617126465,800.0 -200,300,100,800,900,0.20675155520439148,12.001529693603516,0.4244593679904938,800.0 -300,400,100,900,1000,0.18827539682388306,9.276668548583984,0.47026100754737854,800.0 -400,500,100,1000,1100,0.3018724024295807,8.716111183166504,0.4903210401535034,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index 3460a07d3860699b4abdd243afe33b5e41469445..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04719734191894531,18.373680114746094,0.22266511619091034,800.0 -100,200,100,700,800,0.07665779441595078,14.699707984924316,0.30667683482170105,800.0 -200,300,100,800,900,0.12244309484958649,14.447514533996582,0.34635308384895325,800.0 -300,400,100,900,1000,0.10611097514629364,11.942940711975098,0.40486693382263184,800.0 -400,500,100,1000,1100,0.1561962366104126,11.853302001953125,0.4126366674900055,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index 1ad7613771ff81ed79d363fc571a73668d7d4b46..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06265605241060257,17.33863639831543,0.23183241486549377,800.0 -100,200,100,700,800,0.23024456202983856,10.314955711364746,0.42545920610427856,800.0 -200,300,100,800,900,0.18759235739707947,11.376986503601074,0.42710158228874207,800.0 -300,400,100,900,1000,0.15519195795059204,10.047812461853027,0.44462820887565613,800.0 -400,500,100,1000,1100,0.25478821992874146,9.056631088256836,0.4838404953479767,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index 8e36fa99891ea72c369c55d1af55d3ca1cc05ba2..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06243572756648064,17.66242027282715,0.24575139582157135,800.0 -100,200,100,700,800,0.11539915949106216,12.294234275817871,0.36968275904655457,800.0 -200,300,100,800,900,0.109357088804245,14.777706146240234,0.36567336320877075,800.0 -300,400,100,900,1000,0.1634853035211563,10.1857271194458,0.47703537344932556,800.0 -400,500,100,1000,1100,0.22589939832687378,8.36989974975586,0.5018459558486938,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index 97e4b571f2a612e7dfb76b97dc18971444b65cd6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04319189488887787,19.298124313354492,0.19955947995185852,800.0 -100,200,100,700,800,0.12206445634365082,12.558394432067871,0.37400418519973755,800.0 -200,300,100,800,900,0.2028546929359436,12.382981300354004,0.42291459441185,800.0 -300,400,100,900,1000,0.2048528641462326,9.282225608825684,0.45764052867889404,800.0 -400,500,100,1000,1100,0.25466489791870117,9.65378189086914,0.4600163698196411,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index 57c12065de5ec4921ff658b096e034bc21ac2252..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05367765575647354,18.710927963256836,0.2503672242164612,800.0 -100,200,100,700,800,0.060518909245729446,15.504690170288086,0.31254860758781433,800.0 -200,300,100,800,900,0.06899889558553696,16.229290008544922,0.33867499232292175,800.0 -300,400,100,900,1000,0.10057126730680466,12.057714462280273,0.4161822497844696,800.0 -400,500,100,1000,1100,0.08285157382488251,15.50808334350586,0.35378554463386536,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index f2ca935e962bfc122b4554c3571ac47c5162f74d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.08372743427753448,17.44298553466797,0.2504018247127533,800.0 -100,200,100,700,800,0.2112971842288971,9.979231834411621,0.4198806881904602,800.0 -200,300,100,800,900,0.23018695414066315,9.966714859008789,0.4224344491958618,800.0 -300,400,100,900,1000,0.2574789524078369,7.703894138336182,0.4994625747203827,800.0 -400,500,100,1000,1100,0.32337820529937744,6.895928859710693,0.4979221224784851,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index 5917e6ff15eee701c63baaf5c1fc9f8e7e4080dd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04404657334089279,19.037845611572266,0.20887763798236847,800.0 -100,200,100,700,800,0.09402558952569962,14.841216087341309,0.31686267256736755,800.0 -200,300,100,800,900,0.1276928186416626,14.217850685119629,0.34959518909454346,800.0 -300,400,100,900,1000,0.174784317612648,10.425750732421875,0.44521331787109375,800.0 -400,500,100,1000,1100,0.1928538680076599,12.04855728149414,0.42042601108551025,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 57412d994c4c6a1939988adedcb70c9c00b5f44d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04302429035305977,19.309810638427734,0.19046853482723236,800.0 -100,200,100,700,800,0.08574250340461731,13.897473335266113,0.33650368452072144,800.0 -200,300,100,800,900,0.058224115520715714,16.879919052124023,0.32281792163848877,800.0 -300,400,100,900,1000,0.11084021627902985,11.67574691772461,0.4017166197299957,800.0 -400,500,100,1000,1100,0.13929712772369385,12.449746131896973,0.39875441789627075,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index 32961016c14f625697c87ba1abd8ea9d87f338ad..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04625093564391136,19.003084182739258,0.2085171490907669,800.0 -100,200,100,700,800,0.1469363272190094,11.756084442138672,0.39028260111808777,800.0 -200,300,100,800,900,0.15738119184970856,13.193517684936523,0.38451439142227173,800.0 -300,400,100,900,1000,0.16472865641117096,10.152501106262207,0.4413032531738281,800.0 -400,500,100,1000,1100,0.22548151016235352,10.368398666381836,0.43579596281051636,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index 753a6e2bee50448c477dbe8069b02825fc973e47..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06821447610855103,18.248220443725586,0.27413487434387207,800.0 -100,200,100,700,800,0.10116750746965408,14.335022926330566,0.3530313968658447,800.0 -200,300,100,800,900,0.09059583395719528,15.980749130249023,0.34119635820388794,800.0 -300,400,100,900,1000,0.10691726952791214,11.901249885559082,0.40624403953552246,800.0 -400,500,100,1000,1100,0.12964533269405365,12.762215614318848,0.39517223834991455,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index e8cd9afc69da038912718e937eef83ac5387b7f8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04901259392499924,18.72686004638672,0.2243717908859253,800.0 -100,200,100,700,800,0.07678182423114777,15.396592140197754,0.3010353744029999,800.0 -200,300,100,800,900,0.07647860795259476,16.804466247558594,0.30992189049720764,800.0 -300,400,100,900,1000,0.08212831616401672,12.886255264282227,0.37339839339256287,800.0 -400,500,100,1000,1100,0.0734967514872551,14.934662818908691,0.3324694037437439,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index ffba56d8a3fd92d1a2cca1dd52214e71063048bf..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0953756794333458,17.51189422607422,0.25467294454574585,800.0 -100,200,100,700,800,0.19297823309898376,10.957178115844727,0.42034247517585754,800.0 -200,300,100,800,900,0.17086070775985718,13.314716339111328,0.40587303042411804,800.0 -300,400,100,900,1000,0.17165741324424744,9.553638458251953,0.4707811772823334,800.0 -400,500,100,1000,1100,0.25814148783683777,8.633184432983398,0.48357102274894714,800.0 diff --git a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index 66d7c4468ff970467840d7a4e73f849d3ddab58b..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_event_trigger_no_route_anchorclean_a2d4r2_ntok8_8h200_bs16_global128_150k/wandb/offline-run-20260630_192418-t6wroyck/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.059270668774843216,18.631492614746094,0.22987215220928192,800.0 -100,200,100,700,800,0.12883420288562775,13.86412239074707,0.3437853753566742,800.0 -200,300,100,800,900,0.13457125425338745,15.683981895446777,0.34386420249938965,800.0 -300,400,100,900,1000,0.13774864375591278,10.918187141418457,0.42602279782295227,800.0 -400,500,100,1000,1100,0.16095329821109772,11.936484336853027,0.3912748396396637,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 801ff70a6258993267d01d525166d63c9324f830..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06519156694412231,12.938929557800293,0.319004625082016,800.0 -100,200,100,700,800,0.08800570666790009,11.152400016784668,0.39685776829719543,800.0 -200,300,100,800,900,0.10569439828395844,10.14733600616455,0.4393599033355713,800.0 -300,400,100,900,1000,0.10794027149677277,10.088007926940918,0.4442584216594696,800.0 -400,500,100,1000,1100,0.11089298129081726,9.964902877807617,0.5097546577453613,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index 45fff74ad8f12b9dea8c7f11fc8bea866b82b431..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06591605395078659,12.91632080078125,0.3017883896827698,800.0 -100,200,100,700,800,0.08489561825990677,11.376562118530273,0.3706356883049011,800.0 -200,300,100,800,900,0.11591750383377075,10.186568260192871,0.4083808958530426,800.0 -300,400,100,900,1000,0.1002894714474678,10.963298797607422,0.3785412609577179,800.0 -400,500,100,1000,1100,0.10579389333724976,10.306143760681152,0.4231511056423187,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index d5639b6869fd2a872f5fb6ad01198b2e3b38e6c8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.0670163705945015,13.113614082336426,0.30872735381126404,800.0 -100,200,100,700,800,0.07473469525575638,11.920099258422852,0.35016724467277527,800.0 -200,300,100,800,900,0.08144756406545639,11.554858207702637,0.35859498381614685,800.0 -300,400,100,900,1000,0.07385120540857315,12.048540115356445,0.3509782552719116,800.0 -400,500,100,1000,1100,0.08766946941614151,11.259095191955566,0.42828720808029175,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 9f8cb36adf5788d6da4e4013cf05c788c23076db..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06075265258550644,13.275402069091797,0.2904059886932373,800.0 -100,200,100,700,800,0.08628422021865845,11.254912376403809,0.3593354821205139,800.0 -200,300,100,800,900,0.08273929357528687,11.604531288146973,0.3520965576171875,800.0 -300,400,100,900,1000,0.08505845069885254,11.383016586303711,0.34668272733688354,800.0 -400,500,100,1000,1100,0.08835364133119583,11.123225212097168,0.3919909596443176,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 385d9e47dbed4cb988e39e2b40fa2c9acb4a5200..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.059721533209085464,13.608245849609375,0.2754345238208771,800.0 -100,200,100,700,800,0.07228486984968185,12.30798625946045,0.3249071538448334,800.0 -200,300,100,800,900,0.07907383143901825,12.057631492614746,0.3327493965625763,800.0 -300,400,100,900,1000,0.08593664318323135,11.563776969909668,0.34225356578826904,800.0 -400,500,100,1000,1100,0.09548407793045044,10.908727645874023,0.39747297763824463,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 5bb8268fda2a72bec6e9b00759070a241bf4e43c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05977509543299675,13.655170440673828,0.2662762701511383,800.0 -100,200,100,700,800,0.09048973023891449,11.027301788330078,0.3482713997364044,800.0 -200,300,100,800,900,0.08655397593975067,11.611847877502441,0.34612807631492615,800.0 -300,400,100,900,1000,0.10116863995790482,10.632219314575195,0.37663987278938293,800.0 -400,500,100,1000,1100,0.10180144011974335,10.516606330871582,0.39901190996170044,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index 45451c1206521286d4ccfb6da83df3a185fd4e0c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.053853582590818405,13.975768089294434,0.25407329201698303,800.0 -100,200,100,700,800,0.07148709148168564,12.11469841003418,0.3025280833244324,800.0 -200,300,100,800,900,0.06463485956192017,12.793753623962402,0.290566623210907,800.0 -300,400,100,900,1000,0.06666440516710281,12.610730171203613,0.3085559904575348,800.0 -400,500,100,1000,1100,0.08122104406356812,11.91351318359375,0.355396032333374,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 3775fa97c0f981568c6965da055d49ab19ba28f3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.06340260058641434,13.556995391845703,0.27130553126335144,800.0 -100,200,100,700,800,0.06560638546943665,12.729450225830078,0.2938704788684845,800.0 -200,300,100,800,900,0.07434944063425064,12.235712051391602,0.313812792301178,800.0 -300,400,100,900,1000,0.07860279828310013,11.964934349060059,0.3148573935031891,800.0 -400,500,100,1000,1100,0.08312289416790009,11.77541732788086,0.3364546298980713,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index dff028740691b8d00a290ee1b00084d713a0cf12..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.043919771909713745,14.827139854431152,0.21903182566165924,800.0 -100,200,100,700,800,0.054620977491140366,13.390355110168457,0.26259514689445496,800.0 -200,300,100,800,900,0.0560295544564724,13.49074935913086,0.26173949241638184,800.0 -300,400,100,900,1000,0.053635790944099426,13.619487762451172,0.2596305012702942,800.0 -400,500,100,1000,1100,0.06243874505162239,13.065332412719727,0.30696314573287964,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 5026082581ad1b0aa977100a80ef7bad58dbc665..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05098939314484596,14.497206687927246,0.2291307896375656,800.0 -100,200,100,700,800,0.05919884517788887,13.175743103027344,0.2722741663455963,800.0 -200,300,100,800,900,0.07395193725824356,12.417375564575195,0.28403007984161377,800.0 -300,400,100,900,1000,0.07776065170764923,12.351243019104004,0.29426324367523193,800.0 -400,500,100,1000,1100,0.09734532982110977,11.765193939208984,0.3465055823326111,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 48a9e28dc5fac9dc042ef395ced27adb33bb9067..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04505389183759689,15.021700859069824,0.21753500401973724,800.0 -100,200,100,700,800,0.06010746583342552,13.03307819366455,0.27637985348701477,800.0 -200,300,100,800,900,0.05535547807812691,13.8408784866333,0.2701357901096344,800.0 -300,400,100,900,1000,0.05477319657802582,13.55961799621582,0.2832691967487335,800.0 -400,500,100,1000,1100,0.07442670315504074,12.246129989624023,0.3662467300891876,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index 972891098dc15ad86c9b2dbf8188b4ff69e93d01..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03959522396326065,15.442766189575195,0.1980663686990738,800.0 -100,200,100,700,800,0.054308973252773285,13.519400596618652,0.2428179234266281,800.0 -200,300,100,800,900,0.054731860756874084,13.532783508300781,0.2546483278274536,800.0 -300,400,100,900,1000,0.06250152736902237,12.809691429138184,0.27512452006340027,800.0 -400,500,100,1000,1100,0.07368406653404236,12.156350135803223,0.3337048292160034,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index 35a6f403b47631d3cea2a8aaf8cd5a0892d560d4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04034847766160965,15.498046875,0.20045354962348938,800.0 -100,200,100,700,800,0.0616455115377903,13.287541389465332,0.25282394886016846,800.0 -200,300,100,800,900,0.06997858732938766,13.187479019165039,0.2735341191291809,800.0 -300,400,100,900,1000,0.07129544019699097,13.09083366394043,0.2787952423095703,800.0 -400,500,100,1000,1100,0.08613491803407669,12.093908309936523,0.3261650502681732,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index dd402329cd149c3ddb000c83db56969c01f0112c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04510866478085518,15.465160369873047,0.1967311054468155,800.0 -100,200,100,700,800,0.05225163325667381,14.046403884887695,0.24009548127651215,800.0 -200,300,100,800,900,0.060271963477134705,13.681781768798828,0.25579479336738586,800.0 -300,400,100,900,1000,0.06242753937840462,13.32162094116211,0.28128042817115784,800.0 -400,500,100,1000,1100,0.06488361209630966,13.094526290893555,0.2892568111419678,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index c802d100d3af578e31c735458bb77faf2702ad22..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03905917704105377,15.477359771728516,0.1993498057126999,800.0 -100,200,100,700,800,0.048561230301856995,14.297981262207031,0.2388831377029419,800.0 -200,300,100,800,900,0.05899052321910858,13.59373664855957,0.2656235098838806,800.0 -300,400,100,900,1000,0.051312707364559174,13.901433944702148,0.2620603144168854,800.0 -400,500,100,1000,1100,0.06304598599672318,12.992057800292969,0.3059701919555664,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index 4e198c9d3a5c86f9eba8ad1afee362b5fc944605..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04299476742744446,15.582480430603027,0.19524042308330536,800.0 -100,200,100,700,800,0.04943866655230522,13.987980842590332,0.24299995601177216,800.0 -200,300,100,800,900,0.057124581187963486,13.318039894104004,0.27119505405426025,800.0 -300,400,100,900,1000,0.05942675471305847,13.222772598266602,0.28480538725852966,800.0 -400,500,100,1000,1100,0.06640760600566864,12.86728286743164,0.32137787342071533,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index cbd8f7b17f0f8963e012b951280279281d585a64..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.041354384273290634,15.878841400146484,0.18597982823848724,800.0 -100,200,100,700,800,0.04860097914934158,14.391880989074707,0.2305707186460495,800.0 -200,300,100,800,900,0.062167830765247345,13.813823699951172,0.25560450553894043,800.0 -300,400,100,900,1000,0.07572135329246521,12.642151832580566,0.28672125935554504,800.0 -400,500,100,1000,1100,0.11159279197454453,11.33045768737793,0.3599410951137543,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index 0ebacd3f16afb686b5a7cdc2aee01e691754becd..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.04106713458895683,15.65327262878418,0.1973751187324524,800.0 -100,200,100,700,800,0.0457705557346344,14.341543197631836,0.23745571076869965,800.0 -200,300,100,800,900,0.055130086839199066,13.67086410522461,0.2614859640598297,800.0 -300,400,100,900,1000,0.062156036496162415,13.144898414611816,0.28561124205589294,800.0 -400,500,100,1000,1100,0.06660477817058563,12.790382385253906,0.3172571659088135,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 4beba2a9c7a2986029c070fbaa748da962e09e0c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.044022396206855774,15.806958198547363,0.19603927433490753,800.0 -100,200,100,700,800,0.04817481338977814,14.219328880310059,0.23288296163082123,800.0 -200,300,100,800,900,0.0566987507045269,13.575695991516113,0.262421578168869,800.0 -300,400,100,900,1000,0.05047602206468582,14.073010444641113,0.23746536672115326,800.0 -400,500,100,1000,1100,0.06174426153302193,13.202585220336914,0.2893013656139374,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 97e6f60a5018902645fd210ef41f9d657685d2a6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.05800611525774002,14.601300239562988,0.32026225328445435,800.0 -100,200,100,700,800,0.07016308605670929,13.81555461883545,0.37951964139938354,800.0 -200,300,100,800,900,0.10671543329954147,11.610389709472656,0.4041570723056793,800.0 -300,400,100,900,1000,0.11127927154302597,11.234635353088379,0.4619467258453369,800.0 -400,500,100,1000,1100,0.09972650557756424,12.825501441955566,0.4042096734046936,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index b3d747f12c4fda02b68f2cd7dda422a6e50b98d8..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03269733116030693,19.706375122070312,0.2006344199180603,800.0 -100,200,100,700,800,0.05468181520700455,16.185749053955078,0.29474568367004395,800.0 -200,300,100,800,900,0.057473864406347275,17.458396911621094,0.29835808277130127,800.0 -300,400,100,900,1000,0.07460812479257584,13.092613220214844,0.37109634280204773,800.0 -400,500,100,1000,1100,0.060211181640625,14.651025772094727,0.35362184047698975,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 6bb93b8ba230d02d8f52ad834202b7a8247c2070..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03503410890698433,19.807592391967773,0.20826175808906555,800.0 -100,200,100,700,800,0.0715910792350769,15.654850006103516,0.31649020314216614,800.0 -200,300,100,800,900,0.07062074542045593,16.57196617126465,0.31778571009635925,800.0 -300,400,100,900,1000,0.0805869847536087,12.637560844421387,0.4095180630683899,800.0 -400,500,100,1000,1100,0.07798998057842255,14.321564674377441,0.35758131742477417,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 6b2524b6c5133c52c33d32fc92843f25bec270c6..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025987397879362106,20.70648765563965,0.1617702841758728,800.0 -100,200,100,700,800,0.036823708564043045,18.259401321411133,0.21596674621105194,800.0 -200,300,100,800,900,0.050549909472465515,17.89743423461914,0.24205058813095093,800.0 -300,400,100,900,1000,0.06927395612001419,14.179346084594727,0.2929815649986267,800.0 -400,500,100,1000,1100,0.052928585559129715,16.536039352416992,0.2714631259441376,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index e2d2f8d3b70d607d4e32bed35ae255ac522f440a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03347574546933174,20.057645797729492,0.19030877947807312,800.0 -100,200,100,700,800,0.05237233266234398,16.781635284423828,0.2548314034938812,800.0 -200,300,100,800,900,0.06263410300016403,18.17741584777832,0.280877023935318,800.0 -300,400,100,900,1000,0.07832038402557373,13.093171119689941,0.36732473969459534,800.0 -400,500,100,1000,1100,0.04835257679224014,16.606107711791992,0.31585928797721863,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 0c5a7eee753238b533808536afa32debb164de20..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02384866774082184,21.179636001586914,0.15162858366966248,800.0 -100,200,100,700,800,0.03665725886821747,18.563457489013672,0.20551563799381256,800.0 -200,300,100,800,900,0.034242887049913406,19.509187698364258,0.2084498256444931,800.0 -300,400,100,900,1000,0.04538393020629883,15.365017890930176,0.23832067847251892,800.0 -400,500,100,1000,1100,0.035381391644477844,18.71621322631836,0.21876369416713715,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index b8adcf2fabafbb6795530d7f0acf3f7a49d91c33..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.027401577681303024,20.70477294921875,0.1600719839334488,800.0 -100,200,100,700,800,0.03699991852045059,17.89313316345215,0.21284401416778564,800.0 -200,300,100,800,900,0.03660514950752258,19.545801162719727,0.21268315613269806,800.0 -300,400,100,900,1000,0.042554810643196106,15.913732528686523,0.22032783925533295,800.0 -400,500,100,1000,1100,0.02860424295067787,18.94979476928711,0.2058565765619278,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 18d147e35e49187f6be440e0c31054f5ded37c92..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02856428176164627,19.770795822143555,0.16981875896453857,800.0 -100,200,100,700,800,0.0447302870452404,17.130029678344727,0.2262907177209854,800.0 -200,300,100,800,900,0.0672750398516655,16.128976821899414,0.24959416687488556,800.0 -300,400,100,900,1000,0.0938909500837326,12.827994346618652,0.3022192120552063,800.0 -400,500,100,1000,1100,0.08274120092391968,14.957096099853516,0.27598971128463745,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 33f994e9fae4f780a53747a3ed7cef150da05634..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022948790341615677,20.796953201293945,0.15718765556812286,800.0 -100,200,100,700,800,0.03594512492418289,18.520105361938477,0.19748371839523315,800.0 -200,300,100,800,900,0.038082048296928406,19.76667022705078,0.21981912851333618,800.0 -300,400,100,900,1000,0.045159827917814255,16.059249877929688,0.2325512170791626,800.0 -400,500,100,1000,1100,0.029547490179538727,19.07001495361328,0.22039608657360077,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index e21fe3d7c62b569d179011a3a70f503ce0a09f12..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025069452822208405,20.531723022460938,0.14520110189914703,800.0 -100,200,100,700,800,0.047716833651065826,17.538196563720703,0.21745656430721283,800.0 -200,300,100,800,900,0.07447023689746857,16.853368759155273,0.2702763080596924,800.0 -300,400,100,900,1000,0.08246265351772308,13.400708198547363,0.30189260840415955,800.0 -400,500,100,1000,1100,0.0711454227566719,14.951737403869629,0.2845901548862457,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index cf6ceb55c6a243f9b8ea2d1a9df954672ea13a94..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.021982507780194283,21.016027450561523,0.1529441624879837,800.0 -100,200,100,700,800,0.03765752166509628,17.71693992614746,0.2072874903678894,800.0 -200,300,100,800,900,0.05650648474693298,16.123027801513672,0.22636669874191284,800.0 -300,400,100,900,1000,0.058255039155483246,13.89305305480957,0.23949825763702393,800.0 -400,500,100,1000,1100,0.05379787087440491,15.874090194702148,0.2469642609357834,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 8e20fdb756ebfa78d1b12e47b172f6fe43f4bfac..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025494977831840515,20.694726943969727,0.15144824981689453,800.0 -100,200,100,700,800,0.047310829162597656,17.113361358642578,0.21531614661216736,800.0 -200,300,100,800,900,0.07125790417194366,17.959444046020508,0.24329088628292084,800.0 -300,400,100,900,1000,0.07314498722553253,14.9788818359375,0.2656429409980774,800.0 -400,500,100,1000,1100,0.09109305590391159,15.942710876464844,0.2728655934333801,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 8513218c5dc3c7f05f3c5149b7ec058d1a9624fb..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023126613348722458,21.12850570678711,0.14315710961818695,800.0 -100,200,100,700,800,0.057320915162563324,17.772052764892578,0.2134695202112198,800.0 -200,300,100,800,900,0.0690532773733139,17.463960647583008,0.2538180947303772,800.0 -300,400,100,900,1000,0.08422548323869705,14.466052055358887,0.27109992504119873,800.0 -400,500,100,1000,1100,0.07934212684631348,16.222366333007812,0.2584507465362549,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 0fb7afca838f6ffa9882d405bc68ed6fdb01c7a3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02711743302643299,20.731367111206055,0.16786323487758636,800.0 -100,200,100,700,800,0.03440302237868309,18.504648208618164,0.20622722804546356,800.0 -200,300,100,800,900,0.03863105922937393,19.232053756713867,0.22603748738765717,800.0 -300,400,100,900,1000,0.05725853517651558,14.846543312072754,0.29688799381256104,800.0 -400,500,100,1000,1100,0.035951681435108185,18.259910583496094,0.24079066514968872,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 17dde697fea7682d7c616ec65619bab50d325465..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.020641854032874107,21.440710067749023,0.13508689403533936,800.0 -100,200,100,700,800,0.04286869987845421,17.960113525390625,0.21571952104568481,800.0 -200,300,100,800,900,0.03730474412441254,19.207698822021484,0.2240581065416336,800.0 -300,400,100,900,1000,0.05700645595788956,15.26777458190918,0.2678520083427429,800.0 -400,500,100,1000,1100,0.032415010035037994,18.96019744873047,0.22842366993427277,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index a0bd4dc3a7764c3167dae582b0aece93683ae087..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.033568888902664185,19.985628128051758,0.1986057609319687,800.0 -100,200,100,700,800,0.04581310600042343,17.293371200561523,0.2847982347011566,800.0 -200,300,100,800,900,0.056424085050821304,16.695350646972656,0.28736668825149536,800.0 -300,400,100,900,1000,0.07018239051103592,13.915185928344727,0.33484360575675964,800.0 -400,500,100,1000,1100,0.05311264470219612,16.256690979003906,0.29299622774124146,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index f00f03eef574aac5084a40c93eefca995dc40cf5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.031662892550230026,19.976245880126953,0.17300492525100708,800.0 -100,200,100,700,800,0.05323219299316406,17.063682556152344,0.2321784347295761,800.0 -200,300,100,800,900,0.04253438860177994,18.783994674682617,0.2301396131515503,800.0 -300,400,100,900,1000,0.049247242510318756,15.3821439743042,0.24575656652450562,800.0 -400,500,100,1000,1100,0.048011377453804016,17.420574188232422,0.23101137578487396,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index 0fdc3e0eb6e4dfe64cfd1e155caefbc7403a65a5..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02301415428519249,21.049964904785156,0.13353432714939117,800.0 -100,200,100,700,800,0.04307093098759651,18.508691787719727,0.20277538895606995,800.0 -200,300,100,800,900,0.05184818431735039,17.65241050720215,0.22874483466148376,800.0 -300,400,100,900,1000,0.04946098476648331,15.423981666564941,0.22609750926494598,800.0 -400,500,100,1000,1100,0.043139323592185974,16.594392776489258,0.22692112624645233,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 177145fafa77dc84fd5e255f6366a0d0173643a7..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.024452535435557365,20.818880081176758,0.15957261621952057,800.0 -100,200,100,700,800,0.03458739444613457,18.165447235107422,0.20306554436683655,800.0 -200,300,100,800,900,0.04303876310586929,18.530128479003906,0.2315467894077301,800.0 -300,400,100,900,1000,0.05948220193386078,14.645909309387207,0.2814871668815613,800.0 -400,500,100,1000,1100,0.03325784578919411,18.0496883392334,0.22858843207359314,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index b452a93ce9af58c3574fe6a393cabffef1553a7e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02140999026596546,21.460681915283203,0.13596172630786896,800.0 -100,200,100,700,800,0.042557068169116974,18.0057373046875,0.19849653542041779,800.0 -200,300,100,800,900,0.05202772095799446,17.975337982177734,0.22895561158657074,800.0 -300,400,100,900,1000,0.055226441472768784,15.53207778930664,0.24262449145317078,800.0 -400,500,100,1000,1100,0.04550185054540634,17.798601150512695,0.23969368636608124,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index d495d4bc286d43148c53002b13e56e94f70a8491..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02132497727870941,20.940290451049805,0.13463564217090607,800.0 -100,200,100,700,800,0.05652147904038429,17.579669952392578,0.21396030485630035,800.0 -200,300,100,800,900,0.046179044991731644,18.369491577148438,0.22401177883148193,800.0 -300,400,100,900,1000,0.060770291835069656,14.825143814086914,0.2739557921886444,800.0 -400,500,100,1000,1100,0.037481021136045456,17.820219039916992,0.23294557631015778,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index 549d2266781179a4ef502278a47329077bb127ff..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03261321783065796,19.92867660522461,0.17084170877933502,800.0 -100,200,100,700,800,0.049906596541404724,17.126506805419922,0.23150941729545593,800.0 -200,300,100,800,900,0.04808816313743591,18.1998233795166,0.2390270233154297,800.0 -300,400,100,900,1000,0.058495573699474335,15.18945598602295,0.2604570686817169,800.0 -400,500,100,1000,1100,0.03813081607222557,18.13104248046875,0.2390058934688568,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 30fd0d8de3d0dbbfa03f3857e5f7ca8920af2513..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.020970871672034264,21.26675033569336,0.1416231095790863,800.0 -100,200,100,700,800,0.05232065171003342,16.998737335205078,0.2115202397108078,800.0 -200,300,100,800,900,0.0625702440738678,17.868408203125,0.21783959865570068,800.0 -300,400,100,900,1000,0.06737805157899857,15.043896675109863,0.22827266156673431,800.0 -400,500,100,1000,1100,0.04672970622777939,16.802867889404297,0.23399917781352997,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index fc91b600ca78d737ae0a0dcb551098572523583d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02133330889046192,21.057004928588867,0.14180073142051697,800.0 -100,200,100,700,800,0.04374445974826813,18.193750381469727,0.20393162965774536,800.0 -200,300,100,800,900,0.04291291534900665,19.031517028808594,0.2123255580663681,800.0 -300,400,100,900,1000,0.048947688192129135,15.83462142944336,0.2286067008972168,800.0 -400,500,100,1000,1100,0.04853339493274689,17.1335506439209,0.23356442153453827,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index eed8e58ecb231780bfcbd7c522f70ab92a12ad9d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025206202641129494,21.03055191040039,0.13997364044189453,800.0 -100,200,100,700,800,0.03976866230368614,17.564668655395508,0.20892304182052612,800.0 -200,300,100,800,900,0.03744124993681908,18.880252838134766,0.21237991750240326,800.0 -300,400,100,900,1000,0.04739092290401459,15.788810729980469,0.2142665535211563,800.0 -400,500,100,1000,1100,0.02820749580860138,18.878442764282227,0.21140281856060028,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 8ac73804493c9e06483acac8c23578377550e9ae..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.019334299489855766,21.7345027923584,0.13361527025699615,800.0 -100,200,100,700,800,0.03763029724359512,17.94788360595703,0.18960265815258026,800.0 -200,300,100,800,900,0.03992563858628273,18.667692184448242,0.20355607569217682,800.0 -300,400,100,900,1000,0.0519096739590168,15.581368446350098,0.2549022436141968,800.0 -400,500,100,1000,1100,0.029584109783172607,18.89828872680664,0.23866719007492065,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index cee064e84d22614888a7e65597d5eb868db0f28d..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.03232960402965546,18.823312759399414,0.16040951013565063,800.0 -100,200,100,700,800,0.05798763409256935,15.92651653289795,0.2276773601770401,800.0 -200,300,100,800,900,0.05241468548774719,16.951745986938477,0.232399120926857,800.0 -300,400,100,900,1000,0.05468340218067169,14.409897804260254,0.24345897138118744,800.0 -400,500,100,1000,1100,0.06427188217639923,14.973309516906738,0.2570977807044983,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 946de2cef832011c667e62af4f1f77424bcc2d60..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.021341435611248016,21.164936065673828,0.1330205202102661,800.0 -100,200,100,700,800,0.03946204483509064,18.68157386779785,0.1820766031742096,800.0 -200,300,100,800,900,0.06709287315607071,17.629459381103516,0.23595762252807617,800.0 -300,400,100,900,1000,0.08012057095766068,14.959046363830566,0.2546924352645874,800.0 -400,500,100,1000,1100,0.08399640023708344,16.033226013183594,0.2514384984970093,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index 38ddd7c3bf9812169a32d3cba469a0e69903c2fa..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023670710623264313,20.898239135742188,0.14842315018177032,800.0 -100,200,100,700,800,0.05769531801342964,17.20919418334961,0.24738354980945587,800.0 -200,300,100,800,900,0.053262773901224136,18.379859924316406,0.26219743490219116,800.0 -300,400,100,900,1000,0.06092018634080887,15.321250915527344,0.2895241677761078,800.0 -400,500,100,1000,1100,0.05973340943455696,16.90106964111328,0.2916072607040405,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index c01b5a1a29de4407099fae40f31ce5b0637de95f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.019611118361353874,21.047422409057617,0.1373399794101715,800.0 -100,200,100,700,800,0.03725765272974968,18.272741317749023,0.19396589696407318,800.0 -200,300,100,800,900,0.06523435562849045,16.345182418823242,0.2682955265045166,800.0 -300,400,100,900,1000,0.08502187579870224,13.31152629852295,0.3254590630531311,800.0 -400,500,100,1000,1100,0.05639049410820007,15.274428367614746,0.2820408046245575,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index 6af1758d6afc0e3119654116edbd221f19cd6df3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022914888337254524,20.56460952758789,0.15250219404697418,800.0 -100,200,100,700,800,0.0330815427005291,18.61460304260254,0.1883769929409027,800.0 -200,300,100,800,900,0.05299197509884834,17.90160369873047,0.2245410531759262,800.0 -300,400,100,900,1000,0.044161997735500336,15.993968963623047,0.21955335140228271,800.0 -400,500,100,1000,1100,0.03269017115235329,18.26877212524414,0.21411976218223572,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 04783fb9ff845119f033c352a87712a4227e1427..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.021163836121559143,21.159019470214844,0.13627350330352783,800.0 -100,200,100,700,800,0.037396568804979324,18.589622497558594,0.18700343370437622,800.0 -200,300,100,800,900,0.03335227072238922,18.98994255065918,0.20668849349021912,800.0 -300,400,100,900,1000,0.03573572635650635,16.6153621673584,0.19011443853378296,800.0 -400,500,100,1000,1100,0.03139268979430199,18.9877872467041,0.2058117538690567,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index ef3e452ffd1acdb6c68d6940d1a494e223382f3c..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025124475359916687,19.362733840942383,0.14994198083877563,800.0 -100,200,100,700,800,0.06529451906681061,15.120923042297363,0.22064486145973206,800.0 -200,300,100,800,900,0.1239476203918457,12.843707084655762,0.2978827953338623,800.0 -300,400,100,900,1000,0.10614553093910217,11.529736518859863,0.3287089169025421,800.0 -400,500,100,1000,1100,0.15595822036266327,11.167990684509277,0.3624934256076813,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index 8ed1549b2db1c1255d4be88392dc22ea772a6e12..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022892937064170837,20.622825622558594,0.1462627649307251,800.0 -100,200,100,700,800,0.03979836031794548,17.249704360961914,0.2153318077325821,800.0 -200,300,100,800,900,0.04282885417342186,17.730239868164062,0.2330973744392395,800.0 -300,400,100,900,1000,0.04608944058418274,15.789751052856445,0.22238150238990784,800.0 -400,500,100,1000,1100,0.037362415343523026,18.06201934814453,0.2294391691684723,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index 4cf3d1f6e5f4b63b18d5fdaa34b188f149b4232f..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.021639391779899597,20.80266571044922,0.13495047390460968,800.0 -100,200,100,700,800,0.03449353203177452,17.981616973876953,0.1958378553390503,800.0 -200,300,100,800,900,0.03915857896208763,18.347368240356445,0.20779597759246826,800.0 -300,400,100,900,1000,0.04876907169818878,15.735756874084473,0.2232733517885208,800.0 -400,500,100,1000,1100,0.03638351336121559,17.784456253051758,0.21193018555641174,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 80fa3e664ab9f31cebc1c63fe30830aaf3587f87..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023867949843406677,20.827817916870117,0.1393888145685196,800.0 -100,200,100,700,800,0.03614513576030731,18.25833511352539,0.20590323209762573,800.0 -200,300,100,800,900,0.039127759635448456,18.94986915588379,0.24107429385185242,800.0 -300,400,100,900,1000,0.06069446727633476,14.54931354522705,0.30557242035865784,800.0 -400,500,100,1000,1100,0.041028380393981934,18.007783889770508,0.2942037880420685,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 0735a0d8c30238744fdd667d3d45967e85be566a..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.022772016003727913,21.008440017700195,0.1489737629890442,800.0 -100,200,100,700,800,0.0383717305958271,18.479747772216797,0.20901529490947723,800.0 -200,300,100,800,900,0.04155357927083969,18.297630310058594,0.23136450350284576,800.0 -300,400,100,900,1000,0.04828457906842232,15.303967475891113,0.24591709673404694,800.0 -400,500,100,1000,1100,0.04111100733280182,18.356555938720703,0.23633968830108643,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 7605ef680a06c6d18db8f3e2f7ff8b251beafad3..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.021900638937950134,21.31537628173828,0.13578718900680542,800.0 -100,200,100,700,800,0.042063161730766296,18.343778610229492,0.2159149944782257,800.0 -200,300,100,800,900,0.04293680191040039,18.077938079833984,0.24056419730186462,800.0 -300,400,100,900,1000,0.061268072575330734,14.972661972045898,0.28722357749938965,800.0 -400,500,100,1000,1100,0.039312928915023804,18.417510986328125,0.24477188289165497,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 0e77b7ec9f222aab0d14ec937bc2014272f31346..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02209002897143364,21.13543128967285,0.14256349205970764,800.0 -100,200,100,700,800,0.04153624549508095,18.286083221435547,0.19715221226215363,800.0 -200,300,100,800,900,0.04025625064969063,17.56735610961914,0.21875838935375214,800.0 -300,400,100,900,1000,0.05111774057149887,15.158273696899414,0.22227683663368225,800.0 -400,500,100,1000,1100,0.07586982846260071,15.380361557006836,0.26255884766578674,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index ea05d5816731f537dfa39b2d03a9e6038161958e..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.023421650752425194,20.886640548706055,0.1506877839565277,800.0 -100,200,100,700,800,0.046571116894483566,17.80234146118164,0.22302976250648499,800.0 -200,300,100,800,900,0.06072252243757248,17.788999557495117,0.2545630931854248,800.0 -300,400,100,900,1000,0.051105666905641556,15.917095184326172,0.24307380616664886,800.0 -400,500,100,1000,1100,0.06658346205949783,17.29439926147461,0.24850650131702423,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 58460adb7f82a26bc83e7b9286e01cbf2f0c12b4..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.025593996047973633,20.49295425415039,0.1694771647453308,800.0 -100,200,100,700,800,0.04745453968644142,17.660537719726562,0.2210940569639206,800.0 -200,300,100,800,900,0.05088909715414047,17.630664825439453,0.24497058987617493,800.0 -300,400,100,900,1000,0.061335936188697815,15.122014045715332,0.2563353180885315,800.0 -400,500,100,1000,1100,0.058958034962415695,15.930368423461914,0.2580588459968567,800.0 diff --git a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index 2a934874759406eddec57494a73711b79a06c897..0000000000000000000000000000000000000000 --- a/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,100,600,700,0.02093818038702011,21.416858673095703,0.14256158471107483,800.0 -100,200,100,700,800,0.043739404529333115,18.049684524536133,0.2187359631061554,800.0 -200,300,100,800,900,0.043406371027231216,19.183998107910156,0.23073707520961761,800.0 -300,400,100,900,1000,0.05077726021409035,16.302196502685547,0.22290201485157013,800.0 -400,500,100,1000,1100,0.06947535276412964,17.940200805664062,0.24432289600372314,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index f427484551c941bdb559e1e7f6d2ad2c7d85c72a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07437057793140411,12.335322380065918,0.31613659858703613,800.0 -100,200,700,800,0.10054214298725128,10.597856521606445,0.4160316586494446,800.0 -200,300,800,900,0.10484204441308975,10.559826850891113,0.406047523021698,800.0 -300,400,900,1000,0.11591188609600067,9.928277015686035,0.43393126130104065,800.0 -400,500,1000,1100,0.12307212501764297,9.735821723937988,0.4979243576526642,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index e2ce507a89614e7d095f74111269af07ef57b13c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06875681132078171,12.921351432800293,0.30612438917160034,800.0 -100,200,700,800,0.07575023919343948,11.732148170471191,0.35878464579582214,800.0 -200,300,800,900,0.08678798377513885,11.692361831665039,0.36122894287109375,800.0 -300,400,900,1000,0.1010739877820015,10.657149314880371,0.3909285068511963,800.0 -400,500,1000,1100,0.091057188808918,11.136850357055664,0.409728080034256,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 62491dc34f1e786abe2c549bb9e44e314bfc7665..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05726812034845352,13.872397422790527,0.26506102085113525,800.0 -100,200,700,800,0.06975223869085312,12.252232551574707,0.3114652931690216,800.0 -200,300,800,900,0.08392392843961716,11.606855392456055,0.3425321877002716,800.0 -300,400,900,1000,0.07478321343660355,12.311724662780762,0.3038214147090912,800.0 -400,500,1000,1100,0.08697377145290375,11.772063255310059,0.35681870579719543,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index fe5443c5ae20a6eded6a054cadf88b4bc6fc9601..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.053628578782081604,14.132655143737793,0.24989810585975647,800.0 -100,200,700,800,0.06950981914997101,12.463208198547363,0.30199408531188965,800.0 -200,300,800,900,0.08053431659936905,12.074163436889648,0.31963998079299927,800.0 -300,400,900,1000,0.08006837964057922,12.163222312927246,0.3087717294692993,800.0 -400,500,1000,1100,0.08585258573293686,11.746102333068848,0.3427640497684479,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index a792efbfba8f62da06e287f2d201d383e39d4c8a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05569508671760559,13.966609954833984,0.24957388639450073,800.0 -100,200,700,800,0.07250797003507614,12.219343185424805,0.3041503131389618,800.0 -200,300,800,900,0.07637474685907364,12.10986042022705,0.31398627161979675,800.0 -300,400,900,1000,0.07693754136562347,12.094554901123047,0.32765769958496094,800.0 -400,500,1000,1100,0.07718250155448914,11.861091613769531,0.3444455862045288,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index bcd7bb09723e3090e8b11ccc5f715d4f46b1d9c1..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04770689085125923,14.680129051208496,0.22916513681411743,800.0 -100,200,700,800,0.0721101462841034,12.536927223205566,0.2938622236251831,800.0 -200,300,800,900,0.07671350240707397,12.314897537231445,0.3141489326953888,800.0 -300,400,900,1000,0.08031967282295227,12.310866355895996,0.30675575137138367,800.0 -400,500,1000,1100,0.07503923773765564,12.241735458374023,0.3318020701408386,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index bc8d55409e7584bb7bf66d5eac500dbf9574f99f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03856362774968147,15.542646408081055,0.19970867037773132,800.0 -100,200,700,800,0.06692808866500854,12.99151611328125,0.2778869569301605,800.0 -200,300,800,900,0.05445603281259537,13.605927467346191,0.26666733622550964,800.0 -300,400,900,1000,0.06717636436223984,12.836029052734375,0.3011060953140259,800.0 -400,500,1000,1100,0.06992842257022858,12.701062202453613,0.3146783113479614,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 008f75c4ae46defe797b6a3190d62dca7bffbb0a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04356158897280693,14.983436584472656,0.2166198343038559,800.0 -100,200,700,800,0.06134979426860809,13.462518692016602,0.2724597156047821,800.0 -200,300,800,900,0.06358589977025986,13.28744888305664,0.2834659516811371,800.0 -300,400,900,1000,0.0677056536078453,13.110279083251953,0.2941710650920868,800.0 -400,500,1000,1100,0.07739464938640594,12.402678489685059,0.33418983221054077,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index 5ebdda82b5f1e75e109834b5825c90abec5c7a4b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04280576854944229,15.562970161437988,0.2018222063779831,800.0 -100,200,700,800,0.053330183029174805,13.64160442352295,0.25027206540107727,800.0 -200,300,800,900,0.062459178268909454,13.120807647705078,0.2671237885951996,800.0 -300,400,900,1000,0.06262439489364624,13.197247505187988,0.27297842502593994,800.0 -400,500,1000,1100,0.0772109404206276,12.484967231750488,0.3115748465061188,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 55e01fffd1e96683fb89654d476899f747097b20..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03992624208331108,15.690818786621094,0.19532430171966553,800.0 -100,200,700,800,0.05187670886516571,13.9920072555542,0.24563968181610107,800.0 -200,300,800,900,0.056351494044065475,13.795083045959473,0.24910716712474823,800.0 -300,400,900,1000,0.04826008155941963,14.297618865966797,0.23909996449947357,800.0 -400,500,1000,1100,0.062332239001989365,13.45992660522461,0.2911332845687866,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 97b0068da71f08f85925ba7de53c20474fa04ce3..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04967762902379036,15.108442306518555,0.206330344080925,800.0 -100,200,700,800,0.057013411074876785,13.840693473815918,0.2533565163612366,800.0 -200,300,800,900,0.058022595942020416,13.723308563232422,0.25044772028923035,800.0 -300,400,900,1000,0.056321363896131516,13.394831657409668,0.2656201124191284,800.0 -400,500,1000,1100,0.07320602238178253,12.728791236877441,0.32050254940986633,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index a3a67e5bfa7170f6f07c45ff079975d10289d9dc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04187127947807312,15.678618431091309,0.2010483592748642,800.0 -100,200,700,800,0.05109543725848198,14.03752326965332,0.24364742636680603,800.0 -200,300,800,900,0.05654275044798851,13.630820274353027,0.2542611360549927,800.0 -300,400,900,1000,0.05485305190086365,13.63752555847168,0.2505163252353668,800.0 -400,500,1000,1100,0.07321453839540482,12.522871017456055,0.3223523795604706,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index d85d4a383f553c8027ab25037a5ff25f85b9cd3d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03567631542682648,16.342655181884766,0.17942623794078827,800.0 -100,200,700,800,0.05059455707669258,14.252161026000977,0.2307252436876297,800.0 -200,300,800,900,0.05354107916355133,14.379776954650879,0.2370155304670334,800.0 -300,400,900,1000,0.04912323132157326,14.440160751342773,0.22442397475242615,800.0 -400,500,1000,1100,0.06917154788970947,13.0237455368042,0.293249249458313,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index be600e797c54cab3d21cf9831b497ec921d9ba97..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03897200524806976,15.978590965270996,0.1854083240032196,800.0 -100,200,700,800,0.04872000217437744,14.542577743530273,0.22615689039230347,800.0 -200,300,800,900,0.05820280686020851,13.875017166137695,0.24647942185401917,800.0 -300,400,900,1000,0.05333591625094414,14.179930686950684,0.2393760234117508,800.0 -400,500,1000,1100,0.06519737839698792,13.200133323669434,0.28793755173683167,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index 08e4ba769e2c9978cee5b0d416d0d5341c6e3565..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04048103839159012,15.935633659362793,0.18472473323345184,800.0 -100,200,700,800,0.046995896846055984,14.4398193359375,0.22569404542446136,800.0 -200,300,800,900,0.05234915390610695,14.097204208374023,0.23649051785469055,800.0 -300,400,900,1000,0.05465502664446831,13.85224723815918,0.24613040685653687,800.0 -400,500,1000,1100,0.06932605057954788,13.0741548538208,0.30122146010398865,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index 0013dcde300d687a0de1b990db52ab9c7217ac2f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.034585293382406235,16.33797836303711,0.17953701317310333,800.0 -100,200,700,800,0.045884016901254654,14.561784744262695,0.22601157426834106,800.0 -200,300,800,900,0.051806483417749405,14.115556716918945,0.24854591488838196,800.0 -300,400,900,1000,0.054655104875564575,13.69106674194336,0.24383991956710815,800.0 -400,500,1000,1100,0.06373687833547592,13.327914237976074,0.3039604127407074,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index a8aab5d8b940a4b543914d9a9458b74f045d8d31..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0417637825012207,15.788516998291016,0.19876420497894287,800.0 -100,200,700,800,0.0868520736694336,12.569031715393066,0.3164737820625305,800.0 -200,300,800,900,0.10508734732866287,11.822992324829102,0.3315781354904175,800.0 -300,400,900,1000,0.12055038660764694,10.665719032287598,0.3969660997390747,800.0 -400,500,1000,1100,0.12519527971744537,10.618839263916016,0.4113665521144867,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index 0b76cf47bdbe1710964dd1217e0c9d8b6ccc7699..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03598666936159134,16.298898696899414,0.17929381132125854,800.0 -100,200,700,800,0.04530680552124977,14.763427734375,0.21639102697372437,800.0 -200,300,800,900,0.051985401660203934,14.21019172668457,0.23054935038089752,800.0 -300,400,900,1000,0.044979918748140335,14.707493782043457,0.22616428136825562,800.0 -400,500,1000,1100,0.05646006762981415,13.696209907531738,0.2768458127975464,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 7cc561b0734a31d2b7c0940d090c0c9caec646f0..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029937827959656715,16.714597702026367,0.16602925956249237,800.0 -100,200,700,800,0.039301443845033646,15.079626083374023,0.20794066786766052,800.0 -200,300,800,900,0.043980177491903305,14.847759246826172,0.21612663567066193,800.0 -300,400,900,1000,0.04364925995469093,14.776066780090332,0.21580219268798828,800.0 -400,500,1000,1100,0.05649769678711891,13.71115493774414,0.27521148324012756,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index e56756ddb1667fcc3171faf6d850774dde11ebd3..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07257822155952454,13.645618438720703,0.3234003484249115,800.0 -100,200,700,800,0.1628432422876358,10.655309677124023,0.427531898021698,800.0 -200,300,800,900,0.17558583617210388,10.253552436828613,0.40880462527275085,800.0 -300,400,900,1000,0.2308402806520462,9.119914054870605,0.4359395503997803,800.0 -400,500,1000,1100,0.24116264283657074,10.558349609375,0.43542832136154175,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index f66f68fe450b89ca898724bfcf75b9579da825cf..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03498195484280586,20.214570999145508,0.17539848387241364,800.0 -100,200,700,800,0.047542620450258255,17.873825073242188,0.24031026661396027,800.0 -200,300,800,900,0.048730164766311646,17.787870407104492,0.256608247756958,800.0 -300,400,900,1000,0.07252883911132812,13.645118713378906,0.31338509917259216,800.0 -400,500,1000,1100,0.07604958862066269,15.348301887512207,0.312481164932251,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index a8ef1570491af2398ada60c792ba76bb9b7c9969..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03368685767054558,20.27680015563965,0.16947555541992188,800.0 -100,200,700,800,0.048575058579444885,17.095701217651367,0.2630477249622345,800.0 -200,300,800,900,0.04775889962911606,18.37631607055664,0.2514148950576782,800.0 -300,400,900,1000,0.07267235964536667,13.855646133422852,0.32405126094818115,800.0 -400,500,1000,1100,0.06772008538246155,16.085779190063477,0.31098687648773193,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index e6277a7dbba4338ccfe1fe33ce0c9285423de1d9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02905210293829441,20.873291015625,0.15657711029052734,800.0 -100,200,700,800,0.04494656249880791,18.019712448120117,0.21627472341060638,800.0 -200,300,800,900,0.05581386387348175,18.14748191833496,0.2529538869857788,800.0 -300,400,900,1000,0.07803257554769516,13.853079795837402,0.333687961101532,800.0 -400,500,1000,1100,0.05852663144469261,16.596965789794922,0.26373592019081116,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index ac098e3807c755cba981849bc6374466809c6466..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023573188111186028,21.360576629638672,0.14185598492622375,800.0 -100,200,700,800,0.03487637639045715,19.083940505981445,0.2008095234632492,800.0 -200,300,800,900,0.040917087346315384,19.477989196777344,0.21853098273277283,800.0 -300,400,900,1000,0.060904525220394135,15.120072364807129,0.25667664408683777,800.0 -400,500,1000,1100,0.03808801993727684,18.129480361938477,0.24274593591690063,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index af853c054f8b51a3c06ee123dfe1700ab56edcc9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027799751609563828,20.770666122436523,0.16602672636508942,800.0 -100,200,700,800,0.03313408046960831,18.635940551757812,0.20404626429080963,800.0 -200,300,800,900,0.042062874883413315,18.72173309326172,0.23993019759655,800.0 -300,400,900,1000,0.05551965534687042,14.816292762756348,0.26559409499168396,800.0 -400,500,1000,1100,0.07893553376197815,15.42245101928711,0.28717803955078125,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index ef8488e7a708a2b962e57c958a63bc9ba2ca2def..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03698548302054405,20.231101989746094,0.18091292679309845,800.0 -100,200,700,800,0.08175397664308548,16.50139045715332,0.2499258816242218,800.0 -200,300,800,900,0.06088455766439438,18.5084285736084,0.2620936334133148,800.0 -300,400,900,1000,0.0691814124584198,14.635231971740723,0.30550581216812134,800.0 -400,500,1000,1100,0.10825348645448685,14.901381492614746,0.321572482585907,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 6218e9c69d302a621b5185ad1ddc9e3ae79989a6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02890728786587715,20.575302124023438,0.1584080308675766,800.0 -100,200,700,800,0.03705938905477524,19.09465789794922,0.19768862426280975,800.0 -200,300,800,900,0.039365556091070175,19.509794235229492,0.2154705673456192,800.0 -300,400,900,1000,0.04880288243293762,15.546456336975098,0.24829523265361786,800.0 -400,500,1000,1100,0.03551415726542473,17.804866790771484,0.2347947061061859,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 404e842525e858c4a4d67ae6f15fb201b0b03da6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02717401087284088,20.875141143798828,0.1567160189151764,800.0 -100,200,700,800,0.03660262003540993,18.465585708618164,0.2082456797361374,800.0 -200,300,800,900,0.042915213853120804,19.286544799804688,0.24848046898841858,800.0 -300,400,900,1000,0.06498116254806519,14.547293663024902,0.31027698516845703,800.0 -400,500,1000,1100,0.03429245948791504,18.501907348632812,0.23049761354923248,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 7a6de0a1b0074aafecd690614677d45b7f23e6be..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025567281991243362,20.79619598388672,0.15226952731609344,800.0 -100,200,700,800,0.04354197904467583,17.168092727661133,0.2307719588279724,800.0 -200,300,800,900,0.05117557570338249,18.085538864135742,0.2574846148490906,800.0 -300,400,900,1000,0.0627601221203804,14.409658432006836,0.2860940098762512,800.0 -400,500,1000,1100,0.04035211727023125,17.393421173095703,0.2529137134552002,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 9a38708411c1068828a4e95aaa0f95f6813a8460..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023816702887415886,20.78050994873047,0.15571990609169006,800.0 -100,200,700,800,0.038580089807510376,18.499347686767578,0.22177743911743164,800.0 -200,300,800,900,0.04671750217676163,17.78455352783203,0.255039244890213,800.0 -300,400,900,1000,0.06249731034040451,14.298165321350098,0.3099583685398102,800.0 -400,500,1000,1100,0.04457685351371765,16.592374801635742,0.260730117559433,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index ca597e50bae75cc2f509051bac944d1f62e0c570..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02431930974125862,20.599885940551758,0.14878329634666443,800.0 -100,200,700,800,0.04358427971601486,18.235639572143555,0.2051423192024231,800.0 -200,300,800,900,0.04092752933502197,18.929128646850586,0.20644544064998627,800.0 -300,400,900,1000,0.0579562745988369,14.653390884399414,0.27721115946769714,800.0 -400,500,1000,1100,0.03529009222984314,18.321386337280273,0.22986158728599548,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index c2b5669dbf06e0049c2d9a142bb33e912a2d83de..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019844351336359978,21.35770034790039,0.14510560035705566,800.0 -100,200,700,800,0.03216532617807388,18.395233154296875,0.18728506565093994,800.0 -200,300,800,900,0.03069709986448288,19.530088424682617,0.2042633593082428,800.0 -300,400,900,1000,0.044085148721933365,16.139400482177734,0.21670375764369965,800.0 -400,500,1000,1100,0.03810562565922737,17.4340763092041,0.2153037190437317,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 42afc2339a5cfa11847c98d3fff34c7305e8bf7c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02166057378053665,21.400392532348633,0.13442842662334442,800.0 -100,200,700,800,0.0371842160820961,19.193941116333008,0.18541301786899567,800.0 -200,300,800,900,0.03587336465716362,19.39611053466797,0.21087054908275604,800.0 -300,400,900,1000,0.047325216233730316,15.737526893615723,0.22382459044456482,800.0 -400,500,1000,1100,0.051725588738918304,17.123912811279297,0.23742789030075073,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 75d86d3320a2d95d1468593ee05788d1654d3b60..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024767545983195305,20.932641983032227,0.13780833780765533,800.0 -100,200,700,800,0.05688488110899925,17.546716690063477,0.2179289013147354,800.0 -200,300,800,900,0.034252237528562546,19.411540985107422,0.21265655755996704,800.0 -300,400,900,1000,0.05161949619650841,14.91613483428955,0.27573156356811523,800.0 -400,500,1000,1100,0.040217962116003036,17.738866806030273,0.24124212563037872,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 8b7b603e0bcfc81d8d8bf46f1edd197ed43d709c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027180619537830353,20.91567611694336,0.13713249564170837,800.0 -100,200,700,800,0.05155924707651138,17.907207489013672,0.19585925340652466,800.0 -200,300,800,900,0.047166258096694946,18.26702117919922,0.22377771139144897,800.0 -300,400,900,1000,0.06033085659146309,15.059648513793945,0.24672172963619232,800.0 -400,500,1000,1100,0.0861477255821228,14.948724746704102,0.29086220264434814,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index e598aa4145af7660ba5dec8d7761b3359ff7523d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027302196249365807,21.211753845214844,0.1252731829881668,800.0 -100,200,700,800,0.05765816569328308,17.768821716308594,0.20296722650527954,800.0 -200,300,800,900,0.03722720220685005,18.825265884399414,0.20046710968017578,800.0 -300,400,900,1000,0.04338981956243515,16.43930435180664,0.21310478448867798,800.0 -400,500,1000,1100,0.06355121731758118,16.414505004882812,0.24905751645565033,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index 2257d7b82fccb47bc0d13a7f8a5a44a184289c27..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01853974163532257,21.717266082763672,0.11378305405378342,800.0 -100,200,700,800,0.04466637596487999,18.01824378967285,0.1871703863143921,800.0 -200,300,800,900,0.04364766925573349,17.91250991821289,0.22754183411598206,800.0 -300,400,900,1000,0.05527825281023979,15.496478080749512,0.2285260409116745,800.0 -400,500,1000,1100,0.06596808135509491,15.837967872619629,0.2531774044036865,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 9af5f4be6e68180e3290514d7a34f1df46714a5b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02248823083937168,20.627059936523438,0.16280768811702728,800.0 -100,200,700,800,0.04729388281702995,18.108539581298828,0.21959549188613892,800.0 -200,300,800,900,0.057899367064237595,16.84023094177246,0.2552453875541687,800.0 -300,400,900,1000,0.05733036994934082,14.747230529785156,0.26666906476020813,800.0 -400,500,1000,1100,0.048611078411340714,18.312877655029297,0.2339017540216446,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index 6f726b7d18d59b3492d68f0c50db2e431d9bc787..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020158296450972557,21.438186645507812,0.1240781843662262,800.0 -100,200,700,800,0.02958175167441368,19.293237686157227,0.17637723684310913,800.0 -200,300,800,900,0.0294048935174942,19.424468994140625,0.19164921343326569,800.0 -300,400,900,1000,0.0359671488404274,16.826833724975586,0.1769518405199051,800.0 -400,500,1000,1100,0.02304721437394619,19.492813110351562,0.1858333796262741,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index cefb3c00bcb948f51234a08ab811e5cdfbfa0f36..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020168788731098175,21.367979049682617,0.13239805400371552,800.0 -100,200,700,800,0.0335988812148571,19.237091064453125,0.1833948940038681,800.0 -200,300,800,900,0.03348938375711441,19.646398544311523,0.19823865592479706,800.0 -300,400,900,1000,0.039438411593437195,16.196298599243164,0.19911861419677734,800.0 -400,500,1000,1100,0.03810641169548035,18.907878875732422,0.2281273603439331,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index 8cc216ab5d5a3f34c80cfd1f12445bb5d5826588..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02149394527077675,20.70549774169922,0.143443763256073,800.0 -100,200,700,800,0.040162116289138794,18.05895233154297,0.1997520625591278,800.0 -200,300,800,900,0.04400362819433212,17.672218322753906,0.24885524809360504,800.0 -300,400,900,1000,0.0554155558347702,14.984916687011719,0.2758505940437317,800.0 -400,500,1000,1100,0.08047162741422653,15.544134140014648,0.2909972667694092,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 4ffb06bb08e660c12cb133f7b5dd053fe9bb719d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019683193415403366,21.525711059570312,0.1283697485923767,800.0 -100,200,700,800,0.02974633499979973,19.3403377532959,0.1810101717710495,800.0 -200,300,800,900,0.03272140026092529,19.34771728515625,0.19338425993919373,800.0 -300,400,900,1000,0.058153215795755386,15.375405311584473,0.23015518486499786,800.0 -400,500,1000,1100,0.05099727585911751,17.967422485351562,0.2319500893354416,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index a612aeaf1ce435c0f13ea06225ece640a097d436..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01904018595814705,21.4410457611084,0.1376674622297287,800.0 -100,200,700,800,0.033284030854701996,19.039369583129883,0.18538786470890045,800.0 -200,300,800,900,0.04323899745941162,18.419727325439453,0.23155725002288818,800.0 -300,400,900,1000,0.04134134203195572,16.02396011352539,0.22981683909893036,800.0 -400,500,1000,1100,0.03422952815890312,18.186750411987305,0.22611218690872192,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index 309372af3f9f9dec90eb2b9fc5f0995e516372b4..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01979983039200306,21.495927810668945,0.13155028223991394,800.0 -100,200,700,800,0.043468087911605835,18.633445739746094,0.2366991937160492,800.0 -200,300,800,900,0.04337313026189804,17.87701416015625,0.2682293951511383,800.0 -300,400,900,1000,0.06675682961940765,14.604413986206055,0.31163763999938965,800.0 -400,500,1000,1100,0.04368547350168228,17.935787200927734,0.2785385549068451,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index eba534a7e210645979b995a1956b5cd4395e55ec..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0213465616106987,20.847013473510742,0.16725875437259674,800.0 -100,200,700,800,0.039568930864334106,18.710859298706055,0.21785639226436615,800.0 -200,300,800,900,0.04520294815301895,18.536907196044922,0.24742195010185242,800.0 -300,400,900,1000,0.05459856614470482,15.136984825134277,0.2884288430213928,800.0 -400,500,1000,1100,0.03291748836636543,17.9261531829834,0.2530096173286438,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index ac3aa8173dbf77e2af9f905161aab5c576756a60..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027586450800299644,20.222707748413086,0.1520843356847763,800.0 -100,200,700,800,0.03763379156589508,18.224573135375977,0.19958092272281647,800.0 -200,300,800,900,0.04434356093406677,17.94550323486328,0.23812830448150635,800.0 -300,400,900,1000,0.05341808870434761,15.200643539428711,0.25451016426086426,800.0 -400,500,1000,1100,0.034486521035432816,18.034870147705078,0.2245541363954544,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 71f0c18a1d0462fad1dc83f7f20d1b59eed3e67b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01857617311179638,21.689157485961914,0.11518950760364532,800.0 -100,200,700,800,0.03397011756896973,18.969810485839844,0.18024608492851257,800.0 -200,300,800,900,0.035023272037506104,19.171972274780273,0.20224930346012115,800.0 -300,400,900,1000,0.03732879087328911,16.524572372436523,0.1913207620382309,800.0 -400,500,1000,1100,0.02436847612261772,19.60329818725586,0.20089198648929596,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index 52c64d8535458d528d1a2f5ce3a1ef68fcb80679..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022076178342103958,21.014692306518555,0.14255660772323608,800.0 -100,200,700,800,0.03448094427585602,19.262386322021484,0.1750982254743576,800.0 -200,300,800,900,0.03960835561156273,18.19900894165039,0.21398837864398956,800.0 -300,400,900,1000,0.04116252437233925,16.455989837646484,0.21243280172348022,800.0 -400,500,1000,1100,0.025472629815340042,19.1716251373291,0.19414207339286804,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index 8256982b83e3f6af5b7172b3535d3e8c14b09515..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027265185490250587,20.572988510131836,0.14734981954097748,800.0 -100,200,700,800,0.044363442808389664,18.237634658813477,0.19626764953136444,800.0 -200,300,800,900,0.027767343446612358,19.549863815307617,0.18546581268310547,800.0 -300,400,900,1000,0.032727550715208054,17.17441749572754,0.18034294247627258,800.0 -400,500,1000,1100,0.028915241360664368,19.185972213745117,0.1996334046125412,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index d91288b0ea9539a04c0d252b09829fe6c6f5d146..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029213644564151764,20.193227767944336,0.14976966381072998,800.0 -100,200,700,800,0.05767674744129181,17.3880615234375,0.2228771448135376,800.0 -200,300,800,900,0.060316260904073715,16.438745498657227,0.2633601129055023,800.0 -300,400,900,1000,0.06079893931746483,15.723194122314453,0.263655424118042,800.0 -400,500,1000,1100,0.05401046201586723,16.903894424438477,0.26558640599250793,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 7b41f7d123e41ad6e429833a948314574904fd36..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0270851943641901,19.964092254638672,0.16018827259540558,800.0 -100,200,700,800,0.0542948953807354,17.293899536132812,0.23876182734966278,800.0 -200,300,800,900,0.04425697401165962,17.827987670898438,0.2486417442560196,800.0 -300,400,900,1000,0.061134207993745804,14.915146827697754,0.29027360677719116,800.0 -400,500,1000,1100,0.03306495025753975,18.355737686157227,0.2658010423183441,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index 9f4b517f48a26c672cacb9f4368c9fcd0f795ade..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02182803675532341,20.806480407714844,0.13533072173595428,800.0 -100,200,700,800,0.046715859323740005,18.441965103149414,0.1928427368402481,800.0 -200,300,800,900,0.039804231375455856,18.973085403442383,0.21016792953014374,800.0 -300,400,900,1000,0.04739004746079445,16.316898345947266,0.2123209536075592,800.0 -400,500,1000,1100,0.027539562433958054,18.978727340698242,0.19966070353984833,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index 101a1425a6127fa0208f121a25067595a14c62a2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021221943199634552,20.68167495727539,0.1344148963689804,800.0 -100,200,700,800,0.04392356052994728,18.300228118896484,0.2033846080303192,800.0 -200,300,800,900,0.03581675514578819,18.10797882080078,0.22068773210048676,800.0 -300,400,900,1000,0.03860318660736084,16.84223747253418,0.20487745106220245,800.0 -400,500,1000,1100,0.03085242211818695,18.22587013244629,0.2153470814228058,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index 46163de1275811ac9a0d8d1941786d52f0cd97e9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02324836514890194,20.986373901367188,0.1544608771800995,800.0 -100,200,700,800,0.04001319408416748,18.713214874267578,0.1811385601758957,800.0 -200,300,800,900,0.03657018393278122,19.288028717041016,0.20423412322998047,800.0 -300,400,900,1000,0.055188197642564774,16.03200912475586,0.22421762347221375,800.0 -400,500,1000,1100,0.030603675171732903,18.913021087646484,0.22098685801029205,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 89a58f35fee6a69159168777ec2ebc38285bcfe8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02095194347202778,21.391660690307617,0.13471414148807526,800.0 -100,200,700,800,0.03556782007217407,18.902467727661133,0.1763879358768463,800.0 -200,300,800,900,0.03469745069742203,18.894784927368164,0.18971394002437592,800.0 -300,400,900,1000,0.03293322026729584,17.28586196899414,0.16664907336235046,800.0 -400,500,1000,1100,0.02363636903464794,19.753459930419922,0.17890751361846924,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 084d1b9bfcc9e140375780172ee186a0fdcbf839..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020321298390626907,20.711084365844727,0.14556483924388885,800.0 -100,200,700,800,0.03608348220586777,18.793964385986328,0.19940301775932312,800.0 -200,300,800,900,0.03504062443971634,19.1441707611084,0.21162110567092896,800.0 -300,400,900,1000,0.04939115419983864,15.737305641174316,0.2415127009153366,800.0 -400,500,1000,1100,0.04200266674160957,17.49059295654297,0.2247300148010254,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index bb3d3f1d7378e2274fc219b08f479e57b2fdd6e5..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016652075573801994,21.722557067871094,0.10592620819807053,800.0 -100,200,700,800,0.04473917558789253,18.107927322387695,0.21100902557373047,800.0 -200,300,800,900,0.048444557934999466,17.201200485229492,0.2662181854248047,800.0 -300,400,900,1000,0.059298768639564514,14.664643287658691,0.3013768792152405,800.0 -400,500,1000,1100,0.041401058435440063,17.13654327392578,0.276483952999115,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index eb089f6eae3666d5df8b29c8fcbd1a9f667ad4a1..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02146158181130886,21.063796997070312,0.1422005444765091,800.0 -100,200,700,800,0.03971318155527115,18.99834632873535,0.1786704808473587,800.0 -200,300,800,900,0.0448908768594265,18.466157913208008,0.24190063774585724,800.0 -300,400,900,1000,0.06523022800683975,14.936861991882324,0.27308690547943115,800.0 -400,500,1000,1100,0.09882566332817078,15.469793319702148,0.28790852427482605,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index a392e2d1af038e1caf72151fec6a62f94fcea019..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01872427389025688,21.236309051513672,0.1308794915676117,800.0 -100,200,700,800,0.034211549907922745,19.063135147094727,0.17460180819034576,800.0 -200,300,800,900,0.036845747381448746,17.89398193359375,0.18557949364185333,800.0 -300,400,900,1000,0.04999811574816704,16.22809600830078,0.20907871425151825,800.0 -400,500,1000,1100,0.03471548482775688,18.59085464477539,0.21611180901527405,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index deefd6dc6e71230ce8bfde9d9580dc6442dc77c2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01996750198304653,21.382627487182617,0.12861411273479462,800.0 -100,200,700,800,0.04448941722512245,17.540599822998047,0.20293325185775757,800.0 -200,300,800,900,0.05992572009563446,17.607952117919922,0.24103182554244995,800.0 -300,400,900,1000,0.056182898581027985,15.983240127563477,0.2289995551109314,800.0 -400,500,1000,1100,0.05477895587682724,17.504167556762695,0.23429763317108154,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index d70f0bd943d2d3d91267b1d6754823d8ace7ee57..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02026672102510929,20.862648010253906,0.1250646710395813,800.0 -100,200,700,800,0.04540269076824188,17.786788940429688,0.20086880028247833,800.0 -200,300,800,900,0.04282230883836746,18.264921188354492,0.22984512150287628,800.0 -300,400,900,1000,0.04834098741412163,15.702330589294434,0.21678166091442108,800.0 -400,500,1000,1100,0.06358963996171951,16.009685516357422,0.23224954307079315,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index b9e093a672cba888da447db921863ba4afa4a618..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01781855896115303,21.334646224975586,0.11787980794906616,800.0 -100,200,700,800,0.0427454337477684,18.98758316040039,0.18508030474185944,800.0 -200,300,800,900,0.04287967085838318,18.802913665771484,0.2156619280576706,800.0 -300,400,900,1000,0.04655972495675087,16.314964294433594,0.22413690388202667,800.0 -400,500,1000,1100,0.027813054621219635,19.069034576416016,0.2159024029970169,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index 3e783d0687a4ac117c71e3969215078872ce69f9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02579379640519619,20.916919708251953,0.17104904353618622,800.0 -100,200,700,800,0.04345055669546127,18.42563247680664,0.20860472321510315,800.0 -200,300,800,900,0.03668923303484917,19.11046600341797,0.21675905585289001,800.0 -300,400,900,1000,0.04980476200580597,16.005970001220703,0.2213803231716156,800.0 -400,500,1000,1100,0.028231685981154442,18.288223266601562,0.20938414335250854,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index 18d9965e233355f662d02347cc9270875a23a3e2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018763922154903412,21.159950256347656,0.12815718352794647,800.0 -100,200,700,800,0.057792700827121735,16.055326461791992,0.21660029888153076,800.0 -200,300,800,900,0.06239767372608185,16.562776565551758,0.2457035779953003,800.0 -300,400,900,1000,0.058563075959682465,14.704168319702148,0.25731760263442993,800.0 -400,500,1000,1100,0.03963800519704819,17.020450592041016,0.23053161799907684,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index f59eaa8f0240b524a98bd1e70079b574e4f6d38a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023828651756048203,20.250965118408203,0.16306960582733154,800.0 -100,200,700,800,0.048352424055337906,16.644372940063477,0.21811626851558685,800.0 -200,300,800,900,0.05667716637253761,17.398937225341797,0.23394230008125305,800.0 -300,400,900,1000,0.038355179131031036,16.32906150817871,0.19312550127506256,800.0 -400,500,1000,1100,0.034643400460481644,18.328935623168945,0.22537964582443237,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index 7ac108f14cc390ae3653aaa2f9708b32df008cbe..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020653892308473587,21.296363830566406,0.1151125356554985,800.0 -100,200,700,800,0.03253179416060448,18.73785400390625,0.1745961755514145,800.0 -200,300,800,900,0.03738118335604668,18.022380828857422,0.20196251571178436,800.0 -300,400,900,1000,0.038058388978242874,16.23650550842285,0.19044551253318787,800.0 -400,500,1000,1100,0.06392373889684677,15.931377410888672,0.24855336546897888,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index b398932727f54ae9436aca7fca2993a8dcf95c71..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01873668283224106,21.264551162719727,0.1221112608909607,800.0 -100,200,700,800,0.04003113880753517,18.820079803466797,0.17855219542980194,800.0 -200,300,800,900,0.03665713220834732,19.326271057128906,0.18934810161590576,800.0 -300,400,900,1000,0.04278457537293434,16.700664520263672,0.19767625629901886,800.0 -400,500,1000,1100,0.052389830350875854,17.227331161499023,0.23406486213207245,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index f06894947b7bacc639d47bf3f3bd85239bc93596..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015341740101575851,21.85357666015625,0.11723694950342178,800.0 -100,200,700,800,0.02876720018684864,19.655426025390625,0.16198036074638367,800.0 -200,300,800,900,0.03621451184153557,19.34305191040039,0.18966537714004517,800.0 -300,400,900,1000,0.03128649666905403,17.580835342407227,0.16867026686668396,800.0 -400,500,1000,1100,0.0280926376581192,18.871240615844727,0.18000416457653046,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index 8632625bdfacb943f84d7bd5120ae5364ccc82bb..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023003822192549706,20.72064208984375,0.16992202401161194,800.0 -100,200,700,800,0.03307965397834778,19.161813735961914,0.19116167724132538,800.0 -200,300,800,900,0.0398341566324234,18.34804344177246,0.2232479453086853,800.0 -300,400,900,1000,0.0308010745793581,17.307205200195312,0.17072686553001404,800.0 -400,500,1000,1100,0.02683480829000473,19.417940139770508,0.20088939368724823,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 891ba0dec9d2e43b99547e8188962b8781656aee..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017301898449659348,21.615861892700195,0.12619023025035858,800.0 -100,200,700,800,0.04488897696137428,18.86807632446289,0.1938154548406601,800.0 -200,300,800,900,0.03440941870212555,19.35308265686035,0.20248565077781677,800.0 -300,400,900,1000,0.04165757820010185,16.549358367919922,0.20401078462600708,800.0 -400,500,1000,1100,0.034432169049978256,19.188074111938477,0.2099570482969284,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index 76359550975d890c3298215c821584528f6d9468..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019361864775419235,21.564048767089844,0.12547661364078522,800.0 -100,200,700,800,0.031392935663461685,19.251432418823242,0.17878061532974243,800.0 -200,300,800,900,0.028287675231695175,19.90219497680664,0.18295975029468536,800.0 -300,400,900,1000,0.03464781865477562,17.538467407226562,0.17258508503437042,800.0 -400,500,1000,1100,0.023763734847307205,19.666109085083008,0.1978996843099594,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index ed6bec40e07fcb6c06730b21d7e5c2e74e8f0fdc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02064996026456356,20.835487365722656,0.14518950879573822,800.0 -100,200,700,800,0.054911836981773376,18.302419662475586,0.19897955656051636,800.0 -200,300,800,900,0.05120989680290222,17.840017318725586,0.2298007607460022,800.0 -300,400,900,1000,0.04921075701713562,15.79823112487793,0.23428289592266083,800.0 -400,500,1000,1100,0.03451043739914894,17.991825103759766,0.2384175807237625,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index 4f238b3a1c3512fb60de249107a97d6830672666..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025604182854294777,21.17342758178711,0.12488960474729538,800.0 -100,200,700,800,0.06343419849872589,17.90373992919922,0.20327872037887573,800.0 -200,300,800,900,0.036560773849487305,19.291154861450195,0.19517642259597778,800.0 -300,400,900,1000,0.04082464054226875,16.705398559570312,0.17846164107322693,800.0 -400,500,1000,1100,0.027621859684586525,18.507783889770508,0.20405709743499756,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index fbb2984838388cda8ae7da0d87afdbf2636cd31f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017925884574651718,21.665328979492188,0.11495447158813477,800.0 -100,200,700,800,0.050230082124471664,18.412860870361328,0.18512587249279022,800.0 -200,300,800,900,0.07403308153152466,17.629802703857422,0.2100452035665512,800.0 -300,400,900,1000,0.06118471920490265,15.70765495300293,0.1984042227268219,800.0 -400,500,1000,1100,0.05242026224732399,17.437929153442383,0.21948248147964478,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index 43a1110652dc07b68c4a96e8afc8fd68c31aeba8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027136990800499916,20.72378158569336,0.1629544496536255,800.0 -100,200,700,800,0.02831454761326313,19.321697235107422,0.17423737049102783,800.0 -200,300,800,900,0.0335303358733654,18.9277400970459,0.1946018636226654,800.0 -300,400,900,1000,0.03497316315770149,16.504322052001953,0.1923588514328003,800.0 -400,500,1000,1100,0.025756556540727615,19.250648498535156,0.1917755901813507,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00152000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00152000.csv deleted file mode 100644 index b1af80b38e61fda430f05c2b3de136530065b649..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00152000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02067415788769722,20.94562530517578,0.13796694576740265,800.0 -100,200,700,800,0.062052518129348755,17.48058319091797,0.2164289802312851,800.0 -200,300,800,900,0.059874437749385834,17.811946868896484,0.22452636063098907,800.0 -300,400,900,1000,0.055663421750068665,15.807612419128418,0.23294967412948608,800.0 -400,500,1000,1100,0.047296762466430664,17.853469848632812,0.2506815195083618,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00154000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00154000.csv deleted file mode 100644 index 14ca5c635b09e3ec1e476070f6caa5fc7838e074..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00154000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026146508753299713,20.662525177001953,0.16880348324775696,800.0 -100,200,700,800,0.04123249650001526,18.0704402923584,0.2168278843164444,800.0 -200,300,800,900,0.04861835390329361,17.640422821044922,0.24540658295154572,800.0 -300,400,900,1000,0.046467553824186325,15.716565132141113,0.2541062831878662,800.0 -400,500,1000,1100,0.03650261461734772,18.205463409423828,0.23903772234916687,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00156000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00156000.csv deleted file mode 100644 index 1003c8f62eb3c5cc7c2d62650a51e2b2594c7d36..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00156000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02199913002550602,21.16073226928711,0.14046606421470642,800.0 -100,200,700,800,0.03871737793087959,18.452638626098633,0.1964697241783142,800.0 -200,300,800,900,0.04900020733475685,17.430051803588867,0.23226764798164368,800.0 -300,400,900,1000,0.04059814289212227,16.911710739135742,0.20465236902236938,800.0 -400,500,1000,1100,0.04202747344970703,16.765167236328125,0.2499641478061676,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00158000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00158000.csv deleted file mode 100644 index e96be5126ebc3e107b559c0eacac7b447d9744be..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00158000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021908724680542946,20.667051315307617,0.13881273567676544,800.0 -100,200,700,800,0.032329607754945755,19.220914840698242,0.17735892534255981,800.0 -200,300,800,900,0.040568720549345016,19.154010772705078,0.20395688712596893,800.0 -300,400,900,1000,0.04188840091228485,16.66820526123047,0.20189476013183594,800.0 -400,500,1000,1100,0.026835858821868896,18.816667556762695,0.21223819255828857,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00160000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00160000.csv deleted file mode 100644 index e174e922a89ab2eebf6bce8f8093e71d5096f1d4..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00160000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015911178663372993,21.746109008789062,0.10758267343044281,800.0 -100,200,700,800,0.039600543677806854,18.3310604095459,0.1874804049730301,800.0 -200,300,800,900,0.03144102916121483,19.185768127441406,0.19315819442272186,800.0 -300,400,900,1000,0.04268443584442139,16.481107711791992,0.21200008690357208,800.0 -400,500,1000,1100,0.02610042877495289,19.448413848876953,0.21661943197250366,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00162000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00162000.csv deleted file mode 100644 index 8c01ad06521edd9d8aa585db2054749a3cf3013f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00162000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02760763093829155,19.5892333984375,0.1864788979291916,800.0 -100,200,700,800,0.04529820755124092,18.491662979125977,0.2237265706062317,800.0 -200,300,800,900,0.05046624317765236,17.043394088745117,0.28289657831192017,800.0 -300,400,900,1000,0.05449194461107254,14.936456680297852,0.2956259250640869,800.0 -400,500,1000,1100,0.03870904818177223,18.0671329498291,0.26711639761924744,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00164000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00164000.csv deleted file mode 100644 index 4cbed82ff2765522470609e26a0ae76ca2307e40..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00164000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015792159363627434,21.787860870361328,0.11733045428991318,800.0 -100,200,700,800,0.038658205419778824,19.048805236816406,0.19500023126602173,800.0 -200,300,800,900,0.04591698572039604,17.875349044799805,0.21856355667114258,800.0 -300,400,900,1000,0.04460252448916435,16.46084213256836,0.21598653495311737,800.0 -400,500,1000,1100,0.03277088701725006,18.102191925048828,0.24162830412387848,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00166000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00166000.csv deleted file mode 100644 index ad2a66b45d4380a7152a3b25971530287a1c78d9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00166000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02097819745540619,21.241670608520508,0.13716427981853485,800.0 -100,200,700,800,0.030739476904273033,19.124176025390625,0.20343585312366486,800.0 -200,300,800,900,0.040270641446113586,18.32866859436035,0.2497529238462448,800.0 -300,400,900,1000,0.053980082273483276,15.521119117736816,0.26720255613327026,800.0 -400,500,1000,1100,0.029055779799818993,18.849031448364258,0.24250563979148865,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00168000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00168000.csv deleted file mode 100644 index 35ffdf924b3a11f029008fa2087701af215bb44b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00168000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.013813894242048264,22.056480407714844,0.10696282982826233,800.0 -100,200,700,800,0.0435812771320343,18.772096633911133,0.21475732326507568,800.0 -200,300,800,900,0.044275928288698196,17.640851974487305,0.25906264781951904,800.0 -300,400,900,1000,0.04454384744167328,15.715876579284668,0.23889146745204926,800.0 -400,500,1000,1100,0.03068208508193493,18.326374053955078,0.2620130181312561,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00170000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00170000.csv deleted file mode 100644 index 0f690e67aa552d3b3ffd01bc534d7711a22e6a8c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00170000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022015398368239403,21.104351043701172,0.1327071636915207,800.0 -100,200,700,800,0.040658339858055115,18.272239685058594,0.19503502547740936,800.0 -200,300,800,900,0.06411973387002945,17.181188583374023,0.2362367808818817,800.0 -300,400,900,1000,0.036084000021219254,16.698108673095703,0.19228334724903107,800.0 -400,500,1000,1100,0.030839502811431885,18.691383361816406,0.2178450971841812,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00172000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00172000.csv deleted file mode 100644 index 73079c8b63f08959ab612aee50b53e789d732c15..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00172000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022766191512346268,20.892675399780273,0.15825319290161133,800.0 -100,200,700,800,0.03645126894116402,18.22052764892578,0.1983945071697235,800.0 -200,300,800,900,0.034111008048057556,18.413352966308594,0.2072533667087555,800.0 -300,400,900,1000,0.03147315979003906,17.408641815185547,0.16463270783424377,800.0 -400,500,1000,1100,0.021399149671196938,19.921104431152344,0.17987845838069916,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00174000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00174000.csv deleted file mode 100644 index 02f2e1a66b962d973ad0841f9150ff194f43b10f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00174000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02109585329890251,20.722185134887695,0.13296949863433838,800.0 -100,200,700,800,0.04624956101179123,18.647705078125,0.1927093118429184,800.0 -200,300,800,900,0.03696567192673683,18.897653579711914,0.21660374104976654,800.0 -300,400,900,1000,0.04309575632214546,16.708023071289062,0.21689532697200775,800.0 -400,500,1000,1100,0.04881012812256813,17.949073791503906,0.24244068562984467,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00176000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00176000.csv deleted file mode 100644 index 10d2b5dbc40569de1153cc21dcdeab9f256b11b6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00176000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017343956977128983,21.742860794067383,0.10873257368803024,800.0 -100,200,700,800,0.03781683370471001,19.66204261779785,0.16158001124858856,800.0 -200,300,800,900,0.029262997210025787,19.738367080688477,0.18350189924240112,800.0 -300,400,900,1000,0.03412288427352905,17.480100631713867,0.16890254616737366,800.0 -400,500,1000,1100,0.025574207305908203,19.730207443237305,0.19447387754917145,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00178000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00178000.csv deleted file mode 100644 index 5961672f6bdeddee9a66581fdbccfb9f63ab7280..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00178000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.014689871110022068,22.1650447845459,0.099678173661232,800.0 -100,200,700,800,0.02873801998794079,19.701271057128906,0.1660730242729187,800.0 -200,300,800,900,0.035784341394901276,19.54205322265625,0.2000628113746643,800.0 -300,400,900,1000,0.037209510803222656,16.949569702148438,0.17937812209129333,800.0 -400,500,1000,1100,0.028366465121507645,18.927040100097656,0.21057116985321045,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00180000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00180000.csv deleted file mode 100644 index 0ad08fb80ca3fc526157be5688e5901235de225f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00180000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02632170170545578,20.427234649658203,0.13702695071697235,800.0 -100,200,700,800,0.048031896352767944,17.79169273376465,0.205244243144989,800.0 -200,300,800,900,0.039587438106536865,18.301652908325195,0.20999430119991302,800.0 -300,400,900,1000,0.037325818091630936,16.870227813720703,0.19679255783557892,800.0 -400,500,1000,1100,0.03076828643679619,18.955120086669922,0.1996127963066101,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00182000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00182000.csv deleted file mode 100644 index a18b7e4e23a122265c5fc7b5a30a1ad8c0c9b34b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00182000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023831920698285103,20.497243881225586,0.1449872851371765,800.0 -100,200,700,800,0.03873424977064133,18.791318893432617,0.18309158086776733,800.0 -200,300,800,900,0.04359724000096321,17.812049865722656,0.23221205174922943,800.0 -300,400,900,1000,0.05323868617415428,15.6245698928833,0.2561286985874176,800.0 -400,500,1000,1100,0.0313020758330822,18.856435775756836,0.23169225454330444,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00184000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00184000.csv deleted file mode 100644 index d2927c4fc561ac6942e844dd7fa74b9e716f95bd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00184000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01716136559844017,21.67931365966797,0.11683546006679535,800.0 -100,200,700,800,0.02789817377924919,19.146682739257812,0.16557705402374268,800.0 -200,300,800,900,0.03674319386482239,19.528125762939453,0.19630171358585358,800.0 -300,400,900,1000,0.03285057470202446,17.41016387939453,0.16881756484508514,800.0 -400,500,1000,1100,0.025844352319836617,19.662128448486328,0.19042840600013733,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00186000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00186000.csv deleted file mode 100644 index 15ea1ade154b1736593e1351ba8ae8a29d879397..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00186000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023533664643764496,20.788232803344727,0.13363008201122284,800.0 -100,200,700,800,0.044182635843753815,17.7415714263916,0.20686812698841095,800.0 -200,300,800,900,0.047044310718774796,18.038179397583008,0.21259936690330505,800.0 -300,400,900,1000,0.03765173256397247,16.378816604614258,0.1975831538438797,800.0 -400,500,1000,1100,0.049614716321229935,16.129179000854492,0.23695741593837738,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00188000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00188000.csv deleted file mode 100644 index 90100404bc6c8e462be32170eae6e06a41a36639..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00188000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01523029338568449,21.687910079956055,0.12002173066139221,800.0 -100,200,700,800,0.03446722403168678,19.756010055541992,0.1760914921760559,800.0 -200,300,800,900,0.03495345264673233,18.95804786682129,0.1871376633644104,800.0 -300,400,900,1000,0.02987554855644703,17.70123291015625,0.1604660004377365,800.0 -400,500,1000,1100,0.022552020847797394,19.830263137817383,0.1874656081199646,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00190000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00190000.csv deleted file mode 100644 index 8ab1e52fd19a79a5d8da66d7fb00a910bfd0e614..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00190000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019077666103839874,21.218154907226562,0.13555818796157837,800.0 -100,200,700,800,0.03210797533392906,18.958599090576172,0.18027529120445251,800.0 -200,300,800,900,0.03578735142946243,19.062482833862305,0.2184985727071762,800.0 -300,400,900,1000,0.039676032960414886,16.8618221282959,0.21938909590244293,800.0 -400,500,1000,1100,0.03395024314522743,18.737455368041992,0.2390972524881363,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00192000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00192000.csv deleted file mode 100644 index 9f58e791fa025ef58a05ae6b0f9bd6041d35a130..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00192000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01685909926891327,21.430809020996094,0.1113477423787117,800.0 -100,200,700,800,0.03158555552363396,19.434429168701172,0.16875989735126495,800.0 -200,300,800,900,0.03532632067799568,19.408344268798828,0.1861301064491272,800.0 -300,400,900,1000,0.03123626299202442,17.360925674438477,0.17299005389213562,800.0 -400,500,1000,1100,0.024539822712540627,19.963132858276367,0.1837148666381836,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00194000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00194000.csv deleted file mode 100644 index fe428aef601e07a6150874d7a6d97ebee937dc34..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00194000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02226960100233555,20.933483123779297,0.13782377541065216,800.0 -100,200,700,800,0.030252419412136078,19.433584213256836,0.17312316596508026,800.0 -200,300,800,900,0.027331845834851265,19.541540145874023,0.18924009799957275,800.0 -300,400,900,1000,0.03342438116669655,16.82185173034668,0.17918366193771362,800.0 -400,500,1000,1100,0.019419990479946136,20.386152267456055,0.17615807056427002,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00196000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00196000.csv deleted file mode 100644 index a07ac28809344fcc422d56999e58fce3a1b7a2cd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00196000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.08080616593360901,17.270435333251953,0.2132473587989807,800.0 -100,200,700,800,0.1810162365436554,12.209179878234863,0.3925612270832062,800.0 -200,300,800,900,0.22121810913085938,8.804788589477539,0.42702510952949524,800.0 -300,400,900,1000,0.2517441511154175,7.307333469390869,0.4885326325893402,800.0 -400,500,1000,1100,0.32872527837753296,5.1141133308410645,0.5454242825508118,800.0 diff --git a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00198000.csv b/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00198000.csv deleted file mode 100644 index 114504a9da02fefd0a4b2634deec4cb400d2dc73..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_age_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260715_004632-dt027lxn/files/metrics/segment_metrics_step00198000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02198699861764908,20.65680694580078,0.1485598236322403,800.0 -100,200,700,800,0.04024037718772888,18.518037796020508,0.19480979442596436,800.0 -200,300,800,900,0.03882291913032532,18.488523483276367,0.22730372846126556,800.0 -300,400,900,1000,0.0425749309360981,15.692648887634277,0.22505469620227814,800.0 -400,500,1000,1100,0.02971389703452587,18.010404586791992,0.2165384739637375,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index d4fc6038fda9e4cb941c6369df218c1398ed270e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07969200611114502,12.021657943725586,0.37947943806648254,800.0 -100,200,700,800,0.09774310141801834,10.537322044372559,0.4431721568107605,800.0 -200,300,800,900,0.13793522119522095,9.069942474365234,0.5107479691505432,800.0 -300,400,900,1000,0.12975053489208221,9.632528305053711,0.4779835641384125,800.0 -400,500,1000,1100,0.13243965804576874,9.49283218383789,0.4923490881919861,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index 0bd2f91942186ec4db95bf1073fa5116683c63b9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07348881661891937,12.60372543334961,0.33251723647117615,800.0 -100,200,700,800,0.09875397384166718,10.61744499206543,0.40528351068496704,800.0 -200,300,800,900,0.10855261981487274,10.091146469116211,0.4394131600856781,800.0 -300,400,900,1000,0.11764869838953018,9.940312385559082,0.4065725803375244,800.0 -400,500,1000,1100,0.12009724974632263,9.788103103637695,0.45376768708229065,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 27840abe9a63777522ee39472e0bc61e1e7c13c3..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06092148274183273,13.757096290588379,0.2635292708873749,800.0 -100,200,700,800,0.06865542382001877,12.417447090148926,0.31314101815223694,800.0 -200,300,800,900,0.07851392775774002,11.916770935058594,0.3268381357192993,800.0 -300,400,900,1000,0.07895916700363159,11.959692001342773,0.31671372056007385,800.0 -400,500,1000,1100,0.08739742636680603,11.561440467834473,0.36101067066192627,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 57fe4d5cb82d655c437a4379664c6c59cb001bc2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05755767971277237,13.8013916015625,0.25813090801239014,800.0 -100,200,700,800,0.06499183923006058,12.724230766296387,0.301372766494751,800.0 -200,300,800,900,0.07253121584653854,12.537934303283691,0.314383327960968,800.0 -300,400,900,1000,0.07119031250476837,12.550651550292969,0.3112204074859619,800.0 -400,500,1000,1100,0.07338526844978333,12.376175880432129,0.3440695106983185,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 470c276559ea37c2b858326c374b6ba6f1da8ed8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.049151815474033356,14.761909484863281,0.22560188174247742,800.0 -100,200,700,800,0.06317105144262314,12.749795913696289,0.284656286239624,800.0 -200,300,800,900,0.07039617002010345,12.405797958374023,0.2971511781215668,800.0 -300,400,900,1000,0.08166923373937607,11.649998664855957,0.324074923992157,800.0 -400,500,1000,1100,0.08318176120519638,11.619013786315918,0.3433793783187866,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index b654002001141bb007a8dc55b87dd8f7553d5174..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05672835931181908,14.034394264221191,0.24146434664726257,800.0 -100,200,700,800,0.06210148334503174,12.924149513244629,0.2872358560562134,800.0 -200,300,800,900,0.06993032991886139,12.624268531799316,0.291309118270874,800.0 -300,400,900,1000,0.07152644544839859,12.29305648803711,0.2939411997795105,800.0 -400,500,1000,1100,0.0789293497800827,12.126609802246094,0.3394547998905182,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index ebdd2c6bc465e7f2d2be66d402a79c6a9e7d2128..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04907972365617752,14.983187675476074,0.2157006859779358,800.0 -100,200,700,800,0.06433437764644623,12.926752090454102,0.27618056535720825,800.0 -200,300,800,900,0.06721971184015274,13.032270431518555,0.2840418219566345,800.0 -300,400,900,1000,0.0854058638215065,11.893171310424805,0.3236190676689148,800.0 -400,500,1000,1100,0.09427284449338913,11.74328899383545,0.35090529918670654,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 96cc8101aa56d03ecf26bcd442556fe3aa626517..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04661554843187332,15.207538604736328,0.2085576057434082,800.0 -100,200,700,800,0.05282231420278549,13.8360013961792,0.24908547103405,800.0 -200,300,800,900,0.0634099468588829,13.133611679077148,0.27245020866394043,800.0 -300,400,900,1000,0.07069028913974762,12.544951438903809,0.30082839727401733,800.0 -400,500,1000,1100,0.07558447867631912,12.331332206726074,0.3284510672092438,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index 31173eeb7aa3b83e0c159fe7c83f6083b27fec65..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04209911823272705,15.281120300292969,0.20203004777431488,800.0 -100,200,700,800,0.04970913007855415,13.87731647491455,0.23841045796871185,800.0 -200,300,800,900,0.05755758285522461,13.474103927612305,0.2618939280509949,800.0 -300,400,900,1000,0.060838837176561356,12.95650863647461,0.2892840504646301,800.0 -400,500,1000,1100,0.07181324809789658,12.444811820983887,0.33508026599884033,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 7cb55be1ea53880014fb13d5fe5a788d53fe2f27..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04073265567421913,15.739797592163086,0.19512945413589478,800.0 -100,200,700,800,0.05468554422259331,13.832427024841309,0.24499653279781342,800.0 -200,300,800,900,0.0621984489262104,13.556509971618652,0.2561904489994049,800.0 -300,400,900,1000,0.06736839562654495,13.157288551330566,0.2779766917228699,800.0 -400,500,1000,1100,0.07640118896961212,12.79248046875,0.3164655268192291,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index a948e0b2c5fae74368f4270fd398077259e85b5d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04433777928352356,15.767827987670898,0.19313204288482666,800.0 -100,200,700,800,0.0479528084397316,14.280988693237305,0.23139411211013794,800.0 -200,300,800,900,0.05494395270943642,13.954874038696289,0.24182212352752686,800.0 -300,400,900,1000,0.058556556701660156,13.539240837097168,0.25914883613586426,800.0 -400,500,1000,1100,0.06888196617364883,13.065840721130371,0.30402711033821106,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index 6fc0a74ccaa7cb3aa222fafd042c9246c643fc9e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.042974699288606644,15.819042205810547,0.19988073408603668,800.0 -100,200,700,800,0.05880098044872284,13.702951431274414,0.2667528986930847,800.0 -200,300,800,900,0.055736999958753586,13.818744659423828,0.2532180845737457,800.0 -300,400,900,1000,0.06825511157512665,13.194984436035156,0.29641395807266235,800.0 -400,500,1000,1100,0.07920625805854797,12.455920219421387,0.3379966616630554,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index 5ec66d0e574f0a2c397bf2f96efc4d8931a324dc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0435669869184494,15.890966415405273,0.18994422256946564,800.0 -100,200,700,800,0.049784332513809204,14.255451202392578,0.23218795657157898,800.0 -200,300,800,900,0.05663008615374565,14.063157081604004,0.24104736745357513,800.0 -300,400,900,1000,0.055648885667324066,14.054922103881836,0.24286803603172302,800.0 -400,500,1000,1100,0.0696985274553299,12.94205093383789,0.3030068874359131,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index de3e28ba5cfcc4b3769882343d4d3116230c338f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03783195838332176,16.30591583251953,0.1815035194158554,800.0 -100,200,700,800,0.05136977136135101,14.140645980834961,0.2368498295545578,800.0 -200,300,800,900,0.06498977541923523,12.818016052246094,0.27250030636787415,800.0 -300,400,900,1000,0.05454357713460922,13.638879776000977,0.2455310821533203,800.0 -400,500,1000,1100,0.07621992379426956,12.231636047363281,0.3083374798297882,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index 293f3d939579e3ba833f613013e00e573ddb6d78..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0378161296248436,16.03388214111328,0.1856812983751297,800.0 -100,200,700,800,0.045320767909288406,14.738727569580078,0.22255904972553253,800.0 -200,300,800,900,0.060785818845033646,13.75520133972168,0.25324031710624695,800.0 -300,400,900,1000,0.056383538991212845,13.835206031799316,0.251423180103302,800.0 -400,500,1000,1100,0.08545157313346863,11.91676139831543,0.37293770909309387,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index d9ae46f80d479273eea38e85e978f2aee3f69d49..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.046096932142972946,15.868583679199219,0.1942356675863266,800.0 -100,200,700,800,0.06950931251049042,13.267601013183594,0.2968543767929077,800.0 -200,300,800,900,0.07382950931787491,13.167084693908691,0.33158788084983826,800.0 -300,400,900,1000,0.06634736806154251,13.260008811950684,0.34203773736953735,800.0 -400,500,1000,1100,0.08942407369613647,11.819940567016602,0.4230785369873047,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index b045266ba03420b8b6434f71ee6bbabceea43d7f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.037932805716991425,16.2637996673584,0.18746162950992584,800.0 -100,200,700,800,0.05309515446424484,14.365239143371582,0.24756500124931335,800.0 -200,300,800,900,0.06530120223760605,13.705390930175781,0.26060378551483154,800.0 -300,400,900,1000,0.06402769684791565,14.001530647277832,0.2512240707874298,800.0 -400,500,1000,1100,0.07838188856840134,13.214879035949707,0.3096107840538025,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index d4150b6a5478cad822e57e95606dc6fe75df178e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03471909463405609,16.455402374267578,0.17874030768871307,800.0 -100,200,700,800,0.043112412095069885,14.976276397705078,0.21285000443458557,800.0 -200,300,800,900,0.05448761582374573,14.324339866638184,0.23740284144878387,800.0 -300,400,900,1000,0.05094858258962631,14.37024211883545,0.23658569157123566,800.0 -400,500,1000,1100,0.06091844663023949,13.673707008361816,0.27929234504699707,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index a51088ae9f1eb763d920ad36aed2a6f667530f0b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03330071270465851,16.642114639282227,0.17139525711536407,800.0 -100,200,700,800,0.05218746140599251,14.577040672302246,0.22445553541183472,800.0 -200,300,800,900,0.05652201548218727,14.380148887634277,0.23873409628868103,800.0 -300,400,900,1000,0.05311301350593567,14.215097427368164,0.24375678598880768,800.0 -400,500,1000,1100,0.06727617233991623,13.460000991821289,0.2910126745700836,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 282fce9b34a66affde124545fe1dc31c34399cbe..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.08691032230854034,13.15077018737793,0.32038480043411255,800.0 -100,200,700,800,0.13957180082798004,10.372552871704102,0.40233778953552246,800.0 -200,300,800,900,0.10263221710920334,11.478528022766113,0.3471330404281616,800.0 -300,400,900,1000,0.1300189346075058,11.189579010009766,0.3654952347278595,800.0 -400,500,1000,1100,0.14676308631896973,11.114826202392578,0.39970216155052185,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 81ed81e1327d17e1cfcba78656e73e41f3bfa5ef..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02942480333149433,20.390426635742188,0.1576080024242401,800.0 -100,200,700,800,0.03577104210853577,18.854185104370117,0.1929900348186493,800.0 -200,300,800,900,0.03699206933379173,19.231311798095703,0.22612062096595764,800.0 -300,400,900,1000,0.06633546948432922,14.381012916564941,0.3087347745895386,800.0 -400,500,1000,1100,0.042471375316381454,17.50873565673828,0.2590671479701996,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 974b07d89eb1c22ba1fffec250217a673445595e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023740451782941818,20.831987380981445,0.1508704274892807,800.0 -100,200,700,800,0.03463489189743996,18.561935424804688,0.20329712331295013,800.0 -200,300,800,900,0.03812228888273239,19.54721450805664,0.22229982912540436,800.0 -300,400,900,1000,0.04980691894888878,15.032268524169922,0.24366550147533417,800.0 -400,500,1000,1100,0.04140310734510422,17.751384735107422,0.236903116106987,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 4dd8fe6b6a9360b028c8619eea91a4f2be2aece2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02964991331100464,20.550334930419922,0.16632717847824097,800.0 -100,200,700,800,0.03600706160068512,18.49213218688965,0.20977525413036346,800.0 -200,300,800,900,0.04231065884232521,18.857677459716797,0.22902576625347137,800.0 -300,400,900,1000,0.05286099389195442,15.135137557983398,0.26725390553474426,800.0 -400,500,1000,1100,0.0605347640812397,17.291940689086914,0.2586681842803955,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index ad9d97253c1bc9c22cc18733455fe2c421e8c88b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0252681914716959,20.976490020751953,0.14929944276809692,800.0 -100,200,700,800,0.03955753892660141,18.01754379272461,0.22281230986118317,800.0 -200,300,800,900,0.0423574224114418,18.63886070251465,0.24260230362415314,800.0 -300,400,900,1000,0.05966435745358467,14.763813018798828,0.29893216490745544,800.0 -400,500,1000,1100,0.033779967576265335,18.525022506713867,0.2374904751777649,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 6ac87d5d4b1a5322d76e97c826f1d25341c7f12e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02369420975446701,21.05651092529297,0.14331308007240295,800.0 -100,200,700,800,0.04074818268418312,18.656970977783203,0.1965043693780899,800.0 -200,300,800,900,0.04148207977414131,19.567447662353516,0.21849024295806885,800.0 -300,400,900,1000,0.050074122846126556,15.987630844116211,0.23258644342422485,800.0 -400,500,1000,1100,0.0348733551800251,18.706920623779297,0.2307736575603485,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index fe169f61cc7a15eb88ca89ecb002ec2b767b43e9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.033592648804187775,19.93760871887207,0.16750648617744446,800.0 -100,200,700,800,0.043779801577329636,18.398487091064453,0.21354299783706665,800.0 -200,300,800,900,0.038980089128017426,20.025402069091797,0.21652060747146606,800.0 -300,400,900,1000,0.047546859830617905,15.79940414428711,0.24514196813106537,800.0 -400,500,1000,1100,0.046814605593681335,16.584016799926758,0.2630730867385864,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 74fa0dfb6a9041a1fef5720bfc07807465a89394..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026887087151408195,21.280595779418945,0.16357873380184174,800.0 -100,200,700,800,0.03579804673790932,18.69808006286621,0.20804767310619354,800.0 -200,300,800,900,0.04065900668501854,19.240089416503906,0.22475533187389374,800.0 -300,400,900,1000,0.03842519223690033,16.471343994140625,0.19830331206321716,800.0 -400,500,1000,1100,0.0321623720228672,17.699552536010742,0.22056002914905548,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index b5686ce51f5d8633588ef4c4953f0989120ee26f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.028490031138062477,20.393898010253906,0.16023452579975128,800.0 -100,200,700,800,0.04588324949145317,18.417448043823242,0.21504482626914978,800.0 -200,300,800,900,0.0462348572909832,18.773916244506836,0.2512581944465637,800.0 -300,400,900,1000,0.05732142552733421,15.372735977172852,0.2673361301422119,800.0 -400,500,1000,1100,0.07417603582143784,15.024164199829102,0.28639891743659973,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 6203c739e90b65a317395c37df1fb4727e92066e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029048020020127296,20.599594116210938,0.1709933876991272,800.0 -100,200,700,800,0.037792060524225235,17.80086326599121,0.21713930368423462,800.0 -200,300,800,900,0.04250143840909004,18.122716903686523,0.23756994307041168,800.0 -300,400,900,1000,0.05758993700146675,14.618470191955566,0.29094815254211426,800.0 -400,500,1000,1100,0.04544774442911148,17.190685272216797,0.26520514488220215,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 10ecda45fe6142a7886f0761b66a415cb9cbc0d6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02161296457052231,21.550317764282227,0.1375376284122467,800.0 -100,200,700,800,0.03968581557273865,18.677919387817383,0.20395177602767944,800.0 -200,300,800,900,0.041995275765657425,18.068992614746094,0.22011791169643402,800.0 -300,400,900,1000,0.051672182977199554,14.889678001403809,0.26016029715538025,800.0 -400,500,1000,1100,0.06826768815517426,15.847124099731445,0.293813019990921,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 2a49ccd3d01b31a5497e1f2034b066000e945ddf..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020308170467615128,21.579483032226562,0.15517543256282806,800.0 -100,200,700,800,0.03622673824429512,18.742403030395508,0.18897724151611328,800.0 -200,300,800,900,0.031202947720885277,20.00313949584961,0.1915261447429657,800.0 -300,400,900,1000,0.043414611369371414,15.994902610778809,0.19996334612369537,800.0 -400,500,1000,1100,0.027483239769935608,18.86076545715332,0.20777373015880585,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 7558b6d76d88aec984909a2aed1cf855832ce88b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03720466047525406,19.49501609802246,0.1794709414243698,800.0 -100,200,700,800,0.05152711272239685,17.22267723083496,0.2245318740606308,800.0 -200,300,800,900,0.039003077894449234,19.23819923400879,0.2103605717420578,800.0 -300,400,900,1000,0.045632295310497284,16.122926712036133,0.21240168809890747,800.0 -400,500,1000,1100,0.04457602649927139,17.447553634643555,0.24165059626102448,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 190f6a26bc337c0497a5fb920cd9a318a1e5b81b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023880796507000923,21.13446807861328,0.15719513595104218,800.0 -100,200,700,800,0.050619788467884064,17.145050048828125,0.22908645868301392,800.0 -200,300,800,900,0.041245948523283005,18.684528350830078,0.2319236695766449,800.0 -300,400,900,1000,0.056425753980875015,14.927992820739746,0.2615758776664734,800.0 -400,500,1000,1100,0.04396983981132507,17.498870849609375,0.2521442472934723,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index f5c5fb100f14d0ec6328e80de4118ba8c8f573f1..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029807737097144127,19.993722915649414,0.16915732622146606,800.0 -100,200,700,800,0.04007668420672417,17.522714614868164,0.21271991729736328,800.0 -200,300,800,900,0.04305324703454971,17.792583465576172,0.2310531586408615,800.0 -300,400,900,1000,0.04935985058546066,15.502508163452148,0.21029077470302582,800.0 -400,500,1000,1100,0.05414406955242157,16.98837661743164,0.24510912597179413,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 993364d02ed7137552ea42f388382b6f93bfd17c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023523520678281784,20.9222469329834,0.16707085072994232,800.0 -100,200,700,800,0.036859530955553055,19.04938507080078,0.18668325245380402,800.0 -200,300,800,900,0.031845737248659134,19.28209686279297,0.1968001127243042,800.0 -300,400,900,1000,0.038862451910972595,15.990135192871094,0.2129838913679123,800.0 -400,500,1000,1100,0.031979288905858994,18.863100051879883,0.20795486867427826,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index 609b23f84382dbbbc5c7f185e6c41d6b13a11269..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021556980907917023,21.14797592163086,0.14149238169193268,800.0 -100,200,700,800,0.035391904413700104,18.311904907226562,0.20134001970291138,800.0 -200,300,800,900,0.05157451704144478,18.930143356323242,0.22731535136699677,800.0 -300,400,900,1000,0.038688741624355316,16.618751525878906,0.19507811963558197,800.0 -400,500,1000,1100,0.039731722325086594,17.45441246032715,0.23102092742919922,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index a743e1aaf559c0306c46bf6697500444ee5699c5..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021448157727718353,20.993330001831055,0.14414747059345245,800.0 -100,200,700,800,0.0404803641140461,17.71184539794922,0.1992744654417038,800.0 -200,300,800,900,0.03443976119160652,19.457942962646484,0.19987066090106964,800.0 -300,400,900,1000,0.04215171933174133,16.34015464782715,0.19791525602340698,800.0 -400,500,1000,1100,0.033416882157325745,18.08809471130371,0.21454663574695587,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 127ea12afd734ec49f8ff20a61f512e92e6af554..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03572088107466698,19.59114646911621,0.20281009376049042,800.0 -100,200,700,800,0.05370568111538887,17.162321090698242,0.24575519561767578,800.0 -200,300,800,900,0.03782225027680397,18.198476791381836,0.23344454169273376,800.0 -300,400,900,1000,0.05127061530947685,15.34501838684082,0.2652016282081604,800.0 -400,500,1000,1100,0.03849247843027115,18.0377197265625,0.24660947918891907,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index 0d5886912a34f95fa42175321e636ef8081b4036..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023266835138201714,21.1002197265625,0.1461256891489029,800.0 -100,200,700,800,0.03642091155052185,18.51856803894043,0.18732358515262604,800.0 -200,300,800,900,0.038555219769477844,17.652484893798828,0.21229961514472961,800.0 -300,400,900,1000,0.03600945696234703,16.53049087524414,0.19630099833011627,800.0 -400,500,1000,1100,0.03963681682944298,18.13304901123047,0.22010160982608795,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index 1d69551741a1652622c5be037b7ae4062bc2357c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02369963377714157,21.196073532104492,0.14615264534950256,800.0 -100,200,700,800,0.0372948944568634,19.229154586791992,0.1867068111896515,800.0 -200,300,800,900,0.04443422332406044,17.683307647705078,0.20984454452991486,800.0 -300,400,900,1000,0.03658657521009445,16.502260208129883,0.1957617551088333,800.0 -400,500,1000,1100,0.032113369554281235,18.296171188354492,0.20795220136642456,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index 900f5fdb97199473ca550130bd56f0736cc40460..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02420412003993988,20.734230041503906,0.14874452352523804,800.0 -100,200,700,800,0.031723033636808395,19.129770278930664,0.184043750166893,800.0 -200,300,800,900,0.03248068690299988,19.58009910583496,0.19249457120895386,800.0 -300,400,900,1000,0.038615938276052475,16.508499145507812,0.1931459605693817,800.0 -400,500,1000,1100,0.03438977897167206,18.216384887695312,0.20416654646396637,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index dc54533ba73821e235f9f9829a47994c696aa86d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0216049887239933,21.43413543701172,0.13736319541931152,800.0 -100,200,700,800,0.03791286051273346,18.966796875,0.1925060898065567,800.0 -200,300,800,900,0.039088353514671326,19.245298385620117,0.2070009410381317,800.0 -300,400,900,1000,0.046776071190834045,16.38224983215332,0.21787777543067932,800.0 -400,500,1000,1100,0.0313003845512867,18.889665603637695,0.21491698920726776,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index 404597407308e30cef3eb03e3f8d8680bb2b6682..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021212315186858177,21.568334579467773,0.12891945242881775,800.0 -100,200,700,800,0.04025297984480858,18.558433532714844,0.1900038868188858,800.0 -200,300,800,900,0.03436562418937683,19.297029495239258,0.20301900804042816,800.0 -300,400,900,1000,0.040956515818834305,16.55624771118164,0.21132229268550873,800.0 -400,500,1000,1100,0.036579426378011703,18.242630004882812,0.22942571341991425,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index dbb1581260ef958a2bed7f5b11394e64168ad0bf..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0198257677257061,21.39598846435547,0.14451071619987488,800.0 -100,200,700,800,0.03700817748904228,18.30748748779297,0.2098386138677597,800.0 -200,300,800,900,0.04797433316707611,17.616674423217773,0.23988206684589386,800.0 -300,400,900,1000,0.04581531509757042,15.972454071044922,0.23106403648853302,800.0 -400,500,1000,1100,0.03472869470715523,18.443313598632812,0.2397664487361908,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 1d111006200aff8a6ebc37731aa8f09dee1028d7..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020771607756614685,21.00067710876465,0.1434631049633026,800.0 -100,200,700,800,0.03961463272571564,18.943998336791992,0.19517375528812408,800.0 -200,300,800,900,0.04127643629908562,18.839509963989258,0.22614090144634247,800.0 -300,400,900,1000,0.054214511066675186,15.258279800415039,0.27597662806510925,800.0 -400,500,1000,1100,0.03833048418164253,17.602001190185547,0.24685166776180267,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 25c699060d45c64c87e8323b205bd13032814d8e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02474932372570038,20.870393753051758,0.170711949467659,800.0 -100,200,700,800,0.03086506389081478,19.24201202392578,0.18929316103458405,800.0 -200,300,800,900,0.032874565571546555,19.34414291381836,0.20342886447906494,800.0 -300,400,900,1000,0.04531425982713699,16.217666625976562,0.22430258989334106,800.0 -400,500,1000,1100,0.02901984378695488,18.855974197387695,0.21548031270503998,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 7bfe9bbc0ac1ceeff1321c750e09d5c36ee0a58a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020435595884919167,21.27712631225586,0.14827346801757812,800.0 -100,200,700,800,0.03226044028997421,19.195472717285156,0.18013910949230194,800.0 -200,300,800,900,0.032168835401535034,19.191665649414062,0.20082500576972961,800.0 -300,400,900,1000,0.038778308779001236,16.622417449951172,0.18810856342315674,800.0 -400,500,1000,1100,0.025558141991496086,18.79456901550293,0.18838798999786377,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index f08ee0a5057354a34362367ecfe37b69ccd8ddfc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02106214128434658,21.403451919555664,0.12904691696166992,800.0 -100,200,700,800,0.05220725014805794,17.885101318359375,0.20275111496448517,800.0 -200,300,800,900,0.03843555599451065,18.428482055664062,0.2078670859336853,800.0 -300,400,900,1000,0.05302221700549126,15.568901062011719,0.24886682629585266,800.0 -400,500,1000,1100,0.03732317313551903,17.906848907470703,0.249471053481102,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index 93041f700b0b9798a8893e0b0c71e558152b61ab..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03792569041252136,19.910018920898438,0.21678143739700317,800.0 -100,200,700,800,0.052942171692848206,17.694860458374023,0.2517874240875244,800.0 -200,300,800,900,0.054215207695961,17.588769912719727,0.2841503918170929,800.0 -300,400,900,1000,0.0647641122341156,14.273202896118164,0.30219852924346924,800.0 -400,500,1000,1100,0.06812296062707901,14.93030071258545,0.3056093752384186,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index f237a2a85d97ed7e1671abd7153fb1f1eaa593ca..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021640000864863396,21.56865692138672,0.14076469838619232,800.0 -100,200,700,800,0.03636215999722481,18.682659149169922,0.19121672213077545,800.0 -200,300,800,900,0.04397260770201683,18.187231063842773,0.21774034202098846,800.0 -300,400,900,1000,0.04479462280869484,16.166210174560547,0.1920967996120453,800.0 -400,500,1000,1100,0.03887736797332764,17.20047378540039,0.21577247977256775,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 8537196faaa73e88cb6e7e00c76c80295eaa11e4..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018977124243974686,21.716686248779297,0.1278052031993866,800.0 -100,200,700,800,0.03246966749429703,18.948711395263672,0.20164017379283905,800.0 -200,300,800,900,0.036874737590551376,18.766868591308594,0.21844524145126343,800.0 -300,400,900,1000,0.06909067183732986,14.780190467834473,0.2922448515892029,800.0 -400,500,1000,1100,0.03371113911271095,18.366455078125,0.25548064708709717,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index d8c2a3cdb61a69ccfc29c16cc73800466dba0d35..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02207818441092968,21.095348358154297,0.14498376846313477,800.0 -100,200,700,800,0.03657373785972595,19.317087173461914,0.18924656510353088,800.0 -200,300,800,900,0.04837518557906151,18.381141662597656,0.22231465578079224,800.0 -300,400,900,1000,0.05988874286413193,16.031665802001953,0.24104942381381989,800.0 -400,500,1000,1100,0.03479337319731712,18.335224151611328,0.23407740890979767,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index b1c4185c1cca936c87bdd6b4c942db2cbb7d8168..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02439078316092491,21.07681655883789,0.14368519186973572,800.0 -100,200,700,800,0.044664524495601654,18.34697723388672,0.19342608749866486,800.0 -200,300,800,900,0.06081042438745499,17.14960479736328,0.22536201775074005,800.0 -300,400,900,1000,0.04866649582982063,15.651880264282227,0.20484378933906555,800.0 -400,500,1000,1100,0.04037218168377876,17.473594665527344,0.23096565902233124,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index 5b5c4898d76f6c55c1ddedf4050e5903a6320d77..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021613191813230515,21.109909057617188,0.13483096659183502,800.0 -100,200,700,800,0.04033445939421654,19.025426864624023,0.1924341320991516,800.0 -200,300,800,900,0.03995440900325775,18.933734893798828,0.2105410248041153,800.0 -300,400,900,1000,0.06627204269170761,15.630722999572754,0.24676063656806946,800.0 -400,500,1000,1100,0.05019804835319519,17.9101505279541,0.2303796261548996,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 7c7cc23e8d8e4a4230ecee16ec913622841b9bf9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01574990712106228,21.893020629882812,0.12694421410560608,800.0 -100,200,700,800,0.0394902266561985,18.854692459106445,0.19387349486351013,800.0 -200,300,800,900,0.03256915509700775,19.50542449951172,0.20076356828212738,800.0 -300,400,900,1000,0.04196605086326599,16.705551147460938,0.20600324869155884,800.0 -400,500,1000,1100,0.03555170074105263,18.64008331298828,0.20413970947265625,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index bd59739b752e80699afaa06c3a23e6457bb2c4bd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026149895042181015,20.414539337158203,0.14967362582683563,800.0 -100,200,700,800,0.04983728379011154,16.840679168701172,0.22222381830215454,800.0 -200,300,800,900,0.035450149327516556,18.573787689208984,0.21232813596725464,800.0 -300,400,900,1000,0.06204504147171974,14.683490753173828,0.23870940506458282,800.0 -400,500,1000,1100,0.04561638459563255,17.8380069732666,0.2224562019109726,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index dec3a706a2abb6acc7858bcfdc21ebda9441763d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01888185553252697,21.477951049804688,0.12669596076011658,800.0 -100,200,700,800,0.0335208885371685,18.809642791748047,0.18768738210201263,800.0 -200,300,800,900,0.038400206714868546,18.744115829467773,0.22147640585899353,800.0 -300,400,900,1000,0.04351220279932022,16.52912712097168,0.2218792587518692,800.0 -400,500,1000,1100,0.03045949898660183,18.74973487854004,0.2058916985988617,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 134c084f46eb26577a36706dab5b3dda2b57751b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025944340974092484,20.722583770751953,0.14022812247276306,800.0 -100,200,700,800,0.043204233050346375,18.387102127075195,0.18925762176513672,800.0 -200,300,800,900,0.02961641363799572,19.327869415283203,0.1844148188829422,800.0 -300,400,900,1000,0.036741357296705246,16.439739227294922,0.18057040870189667,800.0 -400,500,1000,1100,0.025561463087797165,19.246858596801758,0.18897466361522675,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index b489047888a80a831bc9796f1ea5a318a27011a4..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022567691281437874,21.426054000854492,0.14644649624824524,800.0 -100,200,700,800,0.040719736367464066,19.05229949951172,0.2099037766456604,800.0 -200,300,800,900,0.042066577821969986,18.082077026367188,0.23060227930545807,800.0 -300,400,900,1000,0.04947376251220703,15.867995262145996,0.2209818959236145,800.0 -400,500,1000,1100,0.04605911672115326,18.22869873046875,0.23541559278964996,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index 23eec50f5dc83eb03fa24200d049be59ffe273df..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02384934574365616,20.62051773071289,0.16608144342899323,800.0 -100,200,700,800,0.034057069569826126,19.085861206054688,0.18503057956695557,800.0 -200,300,800,900,0.03130580112338066,19.64861488342285,0.19710935652256012,800.0 -300,400,900,1000,0.032924920320510864,17.367996215820312,0.17359109222888947,800.0 -400,500,1000,1100,0.029735572636127472,19.56538963317871,0.19500596821308136,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index 97367d9e1d3fabcb9806e00a8e1cff3accae283e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0155949043110013,22.07253646850586,0.10714057087898254,800.0 -100,200,700,800,0.04032979533076286,18.896699905395508,0.1713465452194214,800.0 -200,300,800,900,0.041085969656705856,18.288009643554688,0.20332340896129608,800.0 -300,400,900,1000,0.03122178092598915,16.939254760742188,0.16894294321537018,800.0 -400,500,1000,1100,0.03326829895377159,18.08868980407715,0.20928731560707092,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index 45fab0f5e2ce9aaaa41d508b2373a82258a414fd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015649979934096336,21.746437072753906,0.11705973744392395,800.0 -100,200,700,800,0.039037689566612244,18.849170684814453,0.18361148238182068,800.0 -200,300,800,900,0.036133699119091034,19.211002349853516,0.21066632866859436,800.0 -300,400,900,1000,0.0456814207136631,16.598840713500977,0.20815801620483398,800.0 -400,500,1000,1100,0.030086398124694824,19.138385772705078,0.21226882934570312,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index 79a2b9bdd13cbfb3d68efa3bbfb05d21bf5862ff..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02047083154320717,20.916425704956055,0.15008698403835297,800.0 -100,200,700,800,0.03494326397776604,19.334196090698242,0.17547179758548737,800.0 -200,300,800,900,0.037543684244155884,18.772136688232422,0.20812976360321045,800.0 -300,400,900,1000,0.03797120228409767,16.527021408081055,0.18476559221744537,800.0 -400,500,1000,1100,0.05529957637190819,17.484577178955078,0.2205856442451477,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index 8817c9f119932ddb309cb45de86e33efc78ba97d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022254399955272675,20.870832443237305,0.14094769954681396,800.0 -100,200,700,800,0.034775055944919586,18.94586944580078,0.18474040925502777,800.0 -200,300,800,900,0.05248577520251274,18.012104034423828,0.21985526382923126,800.0 -300,400,900,1000,0.044419094920158386,16.487442016601562,0.1935247778892517,800.0 -400,500,1000,1100,0.03822474181652069,17.192340850830078,0.2307104915380478,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index 8171ad31062c93f86571a44e9986d3092f533e53..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02620253711938858,20.634191513061523,0.17135536670684814,800.0 -100,200,700,800,0.03341437876224518,19.01644515991211,0.18405956029891968,800.0 -200,300,800,900,0.040913619101047516,18.1903018951416,0.22698137164115906,800.0 -300,400,900,1000,0.05198575183749199,15.64062213897705,0.27063125371932983,800.0 -400,500,1000,1100,0.03628361597657204,18.067338943481445,0.2058752030134201,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index 278bf59fb6a24085b95ba9d8735e15974e4d7760..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021090805530548096,21.130765914916992,0.13427895307540894,800.0 -100,200,700,800,0.04105522111058235,18.17112159729004,0.20123855769634247,800.0 -200,300,800,900,0.05627400055527687,17.401248931884766,0.249521404504776,800.0 -300,400,900,1000,0.05989650636911392,14.998052597045898,0.28136882185935974,800.0 -400,500,1000,1100,0.04347895458340645,17.68878936767578,0.2755972743034363,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index 551cfec1df370104d2dfdb8743cd6e6773b3f987..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018389608711004257,21.748701095581055,0.11528271436691284,800.0 -100,200,700,800,0.037199653685092926,18.5427188873291,0.18524175882339478,800.0 -200,300,800,900,0.031982503831386566,19.2520809173584,0.19615916907787323,800.0 -300,400,900,1000,0.03832346573472023,16.739137649536133,0.19094039499759674,800.0 -400,500,1000,1100,0.04071303829550743,17.21842384338379,0.22927206754684448,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index 11dafcfa291b4b13ed4b640da1d66fca524fbe44..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0229085274040699,20.938535690307617,0.14699552953243256,800.0 -100,200,700,800,0.036912329494953156,18.697723388671875,0.18452812731266022,800.0 -200,300,800,900,0.07409822940826416,15.732857704162598,0.2614893913269043,800.0 -300,400,900,1000,0.08909614384174347,13.710426330566406,0.2768382728099823,800.0 -400,500,1000,1100,0.04415401443839073,17.56231117248535,0.2380785346031189,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index 675df90f44a21e9747ae9b4cb1d40a24383ca8d5..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020250391215085983,21.077856063842773,0.13593950867652893,800.0 -100,200,700,800,0.0420408770442009,18.497114181518555,0.21757683157920837,800.0 -200,300,800,900,0.046184100210666656,18.402555465698242,0.250857949256897,800.0 -300,400,900,1000,0.05058019980788231,15.44045639038086,0.2507893741130829,800.0 -400,500,1000,1100,0.03392142057418823,18.60883331298828,0.24288654327392578,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 4c76a9ab39b70917b9df7dc77f3c3c617bd727b1..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02209635265171528,21.210426330566406,0.14451390504837036,800.0 -100,200,700,800,0.038827117532491684,18.994903564453125,0.17712627351284027,800.0 -200,300,800,900,0.03673837333917618,19.300861358642578,0.20743487775325775,800.0 -300,400,900,1000,0.037482164800167084,16.890281677246094,0.1907443255186081,800.0 -400,500,1000,1100,0.02998904511332512,18.99447250366211,0.2117139995098114,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index f6689846869db8de25bc97aedae1f994f840ffa8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015956489369273186,21.675756454467773,0.11586267501115799,800.0 -100,200,700,800,0.03931621089577675,18.622385025024414,0.17934270203113556,800.0 -200,300,800,900,0.03514145687222481,18.61005210876465,0.2013748288154602,800.0 -300,400,900,1000,0.03283386677503586,16.892454147338867,0.17773319780826569,800.0 -400,500,1000,1100,0.03352654352784157,18.88934898376465,0.20119427144527435,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index 83e15aa47702b10892230d964460e959b797d428..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021975263953208923,21.355796813964844,0.14195065200328827,800.0 -100,200,700,800,0.03352947160601616,19.183170318603516,0.19292454421520233,800.0 -200,300,800,900,0.032496608793735504,19.659704208374023,0.20175012946128845,800.0 -300,400,900,1000,0.03591834008693695,16.85452651977539,0.19939400255680084,800.0 -400,500,1000,1100,0.033637359738349915,18.510677337646484,0.22240810096263885,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index 7a0e4e974a706eb766690a5aed490bbca5dcf635..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020605718716979027,21.578405380249023,0.12292619049549103,800.0 -100,200,700,800,0.03845375031232834,18.90863800048828,0.17025752365589142,800.0 -200,300,800,900,0.033513832837343216,19.443679809570312,0.19172285497188568,800.0 -300,400,900,1000,0.031430359929800034,17.658573150634766,0.17015615105628967,800.0 -400,500,1000,1100,0.0351080596446991,19.06606674194336,0.20667877793312073,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index aaa546ec845bd139999aff56fcd18373e285507f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016603607684373856,21.6440486907959,0.11495143175125122,800.0 -100,200,700,800,0.02789757214486599,19.541343688964844,0.16489826142787933,800.0 -200,300,800,900,0.02934257499873638,19.677490234375,0.19703590869903564,800.0 -300,400,900,1000,0.043052125722169876,16.38067054748535,0.21090582013130188,800.0 -400,500,1000,1100,0.027496017515659332,19.228666305541992,0.18934068083763123,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index 86f96e130187942ae4eb75f280ef646e5cd74d6d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016552254557609558,21.586679458618164,0.108986034989357,800.0 -100,200,700,800,0.02851707860827446,19.0328369140625,0.1629287153482437,800.0 -200,300,800,900,0.027880726382136345,19.220129013061523,0.19129952788352966,800.0 -300,400,900,1000,0.04033534601330757,16.132144927978516,0.1895962357521057,800.0 -400,500,1000,1100,0.03450678288936615,18.32744789123535,0.21007896959781647,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00152000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00152000.csv deleted file mode 100644 index 7fa6b6ed7951e540ad5837bdf74b2f42c104a7c3..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00152000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01817619614303112,21.67572784423828,0.12833236157894135,800.0 -100,200,700,800,0.023639807477593422,19.994186401367188,0.16536957025527954,800.0 -200,300,800,900,0.040864259004592896,19.39887046813965,0.2030232548713684,800.0 -300,400,900,1000,0.035635605454444885,17.07546615600586,0.17778749763965607,800.0 -400,500,1000,1100,0.031114187091588974,19.180858612060547,0.19105000793933868,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00154000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00154000.csv deleted file mode 100644 index 336375ee9fe43d8842e0949c16ea0d0f9ab30279..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00154000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017305197194218636,21.712627410888672,0.11614511162042618,800.0 -100,200,700,800,0.03679941967129707,19.34524154663086,0.1818104386329651,800.0 -200,300,800,900,0.0424250066280365,18.414569854736328,0.2308729588985443,800.0 -300,400,900,1000,0.04874149337410927,16.064746856689453,0.22585219144821167,800.0 -400,500,1000,1100,0.03005160763859749,18.6340389251709,0.21886223554611206,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00156000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00156000.csv deleted file mode 100644 index 83f91e1dedfbb2821818e08f2f9afcf98020d757..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00156000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016293775290250778,21.43094253540039,0.11979270726442337,800.0 -100,200,700,800,0.04327408969402313,18.149181365966797,0.20628812909126282,800.0 -200,300,800,900,0.03340364992618561,19.0975399017334,0.20697098970413208,800.0 -300,400,900,1000,0.035025663673877716,16.90123748779297,0.18301616609096527,800.0 -400,500,1000,1100,0.03199063614010811,18.654455184936523,0.210242360830307,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00158000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00158000.csv deleted file mode 100644 index 52f82a204ccc038adb166f70279371e859d54ebf..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00158000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018680546432733536,21.106117248535156,0.13166095316410065,800.0 -100,200,700,800,0.026293717324733734,20.009864807128906,0.15517714619636536,800.0 -200,300,800,900,0.026214338839054108,20.022415161132812,0.18625250458717346,800.0 -300,400,900,1000,0.029872309416532516,17.651042938232422,0.16399095952510834,800.0 -400,500,1000,1100,0.030671602115035057,19.133323669433594,0.1915445476770401,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00160000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00160000.csv deleted file mode 100644 index de4ef38f95f3232e3d7f8d8ab47a406b768a3f31..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00160000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021244270727038383,21.292800903320312,0.1333232820034027,800.0 -100,200,700,800,0.02886801026761532,19.66275405883789,0.1729276329278946,800.0 -200,300,800,900,0.036356110125780106,18.80634117126465,0.20075128972530365,800.0 -300,400,900,1000,0.03600545972585678,17.0452938079834,0.17250266671180725,800.0 -400,500,1000,1100,0.03499121963977814,18.221166610717773,0.21089643239974976,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00162000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00162000.csv deleted file mode 100644 index ac522cdc87cec8a1f1c147deda421f32a8655756..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00162000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020308377221226692,20.936452865600586,0.14350351691246033,800.0 -100,200,700,800,0.035957228392362595,18.987573623657227,0.20117901265621185,800.0 -200,300,800,900,0.03808636590838432,19.047508239746094,0.21770420670509338,800.0 -300,400,900,1000,0.04086494818329811,16.47871971130371,0.1994638293981552,800.0 -400,500,1000,1100,0.044460926204919815,17.553386688232422,0.26137569546699524,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00164000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00164000.csv deleted file mode 100644 index d38d5cba9277c9a1d99d112ef588ab06dbb5be73..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00164000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020172545686364174,21.241722106933594,0.12994888424873352,800.0 -100,200,700,800,0.029967421665787697,19.398147583007812,0.16790226101875305,800.0 -200,300,800,900,0.034420911222696304,19.153884887695312,0.19541847705841064,800.0 -300,400,900,1000,0.03528488799929619,17.24178123474121,0.18700435757637024,800.0 -400,500,1000,1100,0.03553841635584831,18.847652435302734,0.2099914699792862,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00166000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00166000.csv deleted file mode 100644 index 96609a82d39e5d299e80f35cd45758d0b5c3a980..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00166000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.014964094385504723,21.9506778717041,0.11318185925483704,800.0 -100,200,700,800,0.030630383640527725,19.700698852539062,0.1595039665699005,800.0 -200,300,800,900,0.035120073705911636,19.55213165283203,0.2047361433506012,800.0 -300,400,900,1000,0.036517880856990814,17.341001510620117,0.18266986310482025,800.0 -400,500,1000,1100,0.04036848619580269,19.260835647583008,0.21351662278175354,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00168000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00168000.csv deleted file mode 100644 index 3046a136078d6e9a579fa3b3918e5e55275b8e29..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00168000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01909174956381321,21.529518127441406,0.1245545968413353,800.0 -100,200,700,800,0.02878030575811863,19.44198989868164,0.17102935910224915,800.0 -200,300,800,900,0.030161544680595398,19.624387741088867,0.1925549954175949,800.0 -300,400,900,1000,0.03606603294610977,17.27587127685547,0.18916818499565125,800.0 -400,500,1000,1100,0.026193756610155106,19.880840301513672,0.20838887989521027,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00170000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00170000.csv deleted file mode 100644 index 3c64478765cf4246c896068c14563d0aebedf9fd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00170000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017132330685853958,21.579933166503906,0.1196540892124176,800.0 -100,200,700,800,0.02536259964108467,19.650341033935547,0.16392476856708527,800.0 -200,300,800,900,0.025231005623936653,20.103683471679688,0.17744342982769012,800.0 -300,400,900,1000,0.03458065167069435,17.40514373779297,0.17067062854766846,800.0 -400,500,1000,1100,0.022255998104810715,19.786108016967773,0.17760103940963745,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00172000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00172000.csv deleted file mode 100644 index ecc8c737313e61fe484476f9fbb66b02ca10026c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00172000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021126490086317062,21.03154754638672,0.1431751698255539,800.0 -100,200,700,800,0.03317384794354439,18.495054244995117,0.18783724308013916,800.0 -200,300,800,900,0.037412576377391815,18.846782684326172,0.21498249471187592,800.0 -300,400,900,1000,0.03786808252334595,17.20331573486328,0.18841791152954102,800.0 -400,500,1000,1100,0.04521094262599945,16.434289932250977,0.2497273087501526,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00174000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00174000.csv deleted file mode 100644 index c670709bf47d8ee477b885faaf027e324ca952bc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00174000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027093302458524704,20.471847534179688,0.162589892745018,800.0 -100,200,700,800,0.04385519027709961,18.326313018798828,0.19514016807079315,800.0 -200,300,800,900,0.031980812549591064,19.61089515686035,0.21230833232402802,800.0 -300,400,900,1000,0.040019597858190536,16.304460525512695,0.2245377153158188,800.0 -400,500,1000,1100,0.04719315469264984,18.173383712768555,0.26192742586135864,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00176000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00176000.csv deleted file mode 100644 index 5c8f409095c0b132b0c70445d8066caf950761ee..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00176000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019702492281794548,21.637548446655273,0.1277657151222229,800.0 -100,200,700,800,0.0405406653881073,18.88595199584961,0.18286201357841492,800.0 -200,300,800,900,0.03355953097343445,19.426223754882812,0.19782760739326477,800.0 -300,400,900,1000,0.03375222533941269,17.478992462158203,0.16333884000778198,800.0 -400,500,1000,1100,0.03804372623562813,18.247278213500977,0.2108391374349594,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00178000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00178000.csv deleted file mode 100644 index d0f3b82eaaff14f85a2dddc8300309277abefe40..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00178000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020068373531103134,21.234079360961914,0.1333005726337433,800.0 -100,200,700,800,0.03403010964393616,18.604633331298828,0.17795100808143616,800.0 -200,300,800,900,0.03377043828368187,19.1357479095459,0.20078586041927338,800.0 -300,400,900,1000,0.035790666937828064,16.82313346862793,0.1989605724811554,800.0 -400,500,1000,1100,0.033444520086050034,18.546262741088867,0.21792957186698914,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00180000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00180000.csv deleted file mode 100644 index 278907e5ff1945e102c4ac4cac06d9746935d53c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00180000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025325503200292587,20.632577896118164,0.1533481776714325,800.0 -100,200,700,800,0.03247685730457306,19.33490753173828,0.1828445941209793,800.0 -200,300,800,900,0.03644620254635811,19.302637100219727,0.1951657235622406,800.0 -300,400,900,1000,0.03834802657365799,16.98072052001953,0.18769235908985138,800.0 -400,500,1000,1100,0.03100540116429329,18.814502716064453,0.21412542462348938,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00182000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00182000.csv deleted file mode 100644 index 7fb2db606835c561a8f55246bc65874c81dd171e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00182000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015804069116711617,21.968650817871094,0.11299985647201538,800.0 -100,200,700,800,0.04133416339755058,18.68183708190918,0.190170019865036,800.0 -200,300,800,900,0.038599319756031036,18.721261978149414,0.19975565373897552,800.0 -300,400,900,1000,0.044170890003442764,16.4993953704834,0.20021934807300568,800.0 -400,500,1000,1100,0.03692608326673508,18.639785766601562,0.21135900914669037,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00184000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00184000.csv deleted file mode 100644 index 6392814c7fa027a81f1fca069ecc01c8dee00f14..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00184000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016848914325237274,21.896018981933594,0.11207649111747742,800.0 -100,200,700,800,0.029669566079974174,19.944355010986328,0.14602796733379364,800.0 -200,300,800,900,0.043057750910520554,17.597509384155273,0.20820991694927216,800.0 -300,400,900,1000,0.039381738752126694,16.433629989624023,0.19070380926132202,800.0 -400,500,1000,1100,0.032150860875844955,18.21122932434082,0.20891037583351135,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00186000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00186000.csv deleted file mode 100644 index fd7fd28388d2cad07b0f2ce46aa06aa761191f50..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00186000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015399322845041752,21.874300003051758,0.10738986730575562,800.0 -100,200,700,800,0.030928490683436394,19.624797821044922,0.1586921066045761,800.0 -200,300,800,900,0.035745371133089066,19.311058044433594,0.17892496287822723,800.0 -300,400,900,1000,0.03172624856233597,17.53942108154297,0.1630379855632782,800.0 -400,500,1000,1100,0.021661529317498207,19.9196720123291,0.16396309435367584,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00188000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00188000.csv deleted file mode 100644 index 48c3389e71ff805e77c3978827d29e7e83e05c5b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00188000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020104708150029182,21.66384506225586,0.11447079479694366,800.0 -100,200,700,800,0.035942304879426956,19.307722091674805,0.1680532991886139,800.0 -200,300,800,900,0.03796710446476936,18.933061599731445,0.19567584991455078,800.0 -300,400,900,1000,0.03185565397143364,17.662044525146484,0.16514140367507935,800.0 -400,500,1000,1100,0.025848565623164177,19.380970001220703,0.17737989127635956,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00190000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00190000.csv deleted file mode 100644 index 8992a9903da11454eafb6a41873a12e0ea5f4ac6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00190000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02761482261121273,20.31593132019043,0.15516552329063416,800.0 -100,200,700,800,0.046335428953170776,17.40904998779297,0.21978996694087982,800.0 -200,300,800,900,0.04277868568897247,18.082914352416992,0.23015953600406647,800.0 -300,400,900,1000,0.05920729786157608,15.345664024353027,0.2596703767776489,800.0 -400,500,1000,1100,0.04707002639770508,16.79473114013672,0.2811915874481201,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00192000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00192000.csv deleted file mode 100644 index 19cc06ffda8b0ee047e71f40567671841d6540b2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00192000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016631493344902992,21.74884605407715,0.12040331214666367,800.0 -100,200,700,800,0.03450021892786026,19.526992797851562,0.1779453605413437,800.0 -200,300,800,900,0.03727841377258301,18.727582931518555,0.20211631059646606,800.0 -300,400,900,1000,0.03579161316156387,17.0725040435791,0.18972019851207733,800.0 -400,500,1000,1100,0.04128902778029442,18.129783630371094,0.2261197715997696,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00194000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00194000.csv deleted file mode 100644 index 74cce7189ba24763683902d2c2bcc4e4196a6e66..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00194000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020439602434635162,21.35736846923828,0.13105854392051697,800.0 -100,200,700,800,0.03902812302112579,18.92624282836914,0.18022853136062622,800.0 -200,300,800,900,0.03221068158745766,19.639671325683594,0.18225671350955963,800.0 -300,400,900,1000,0.042009200900793076,16.786779403686523,0.1802145391702652,800.0 -400,500,1000,1100,0.04556174576282501,18.23540687561035,0.21739786863327026,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00196000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00196000.csv deleted file mode 100644 index ae1cb7f2d86867dd57f3c7fde33f5397458e6cec..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00196000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03133082017302513,20.529626846313477,0.16277973353862762,800.0 -100,200,700,800,0.04545390233397484,18.9340877532959,0.1989651322364807,800.0 -200,300,800,900,0.08161384612321854,16.590404510498047,0.26718902587890625,800.0 -300,400,900,1000,0.08009389787912369,15.061982154846191,0.25186634063720703,800.0 -400,500,1000,1100,0.06541933864355087,16.208559036254883,0.2548588514328003,800.0 diff --git a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00198000.csv b/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00198000.csv deleted file mode 100644 index c7c0eaaf20034264be1fe09dc51f716ae181b6a1..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_noise_off_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260717_202138-5pi805i1/files/metrics/segment_metrics_step00198000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02818864770233631,20.34351921081543,0.16850213706493378,800.0 -100,200,700,800,0.0382649190723896,19.035581588745117,0.1874544471502304,800.0 -200,300,800,900,0.05333612486720085,17.586406707763672,0.23262713849544525,800.0 -300,400,900,1000,0.05662660673260689,15.647414207458496,0.22705203294754028,800.0 -400,500,1000,1100,0.03957166522741318,17.672771453857422,0.21728108823299408,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 6cb0c390b2aff529088dd730d2145f18c868e706..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07635684311389923,12.182282447814941,0.32623976469039917,800.0 -100,200,700,800,0.0987328439950943,10.72092056274414,0.4118029773235321,800.0 -200,300,800,900,0.10307610780000687,10.590137481689453,0.40910887718200684,800.0 -300,400,900,1000,0.10791724920272827,10.291062355041504,0.4280160665512085,800.0 -400,500,1000,1100,0.10167224705219269,10.411696434020996,0.45746275782585144,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index 4636385e6ca8c25d65adc0b20c20f7568beb6470..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06823115795850754,12.864631652832031,0.31309056282043457,800.0 -100,200,700,800,0.08155125379562378,11.443358421325684,0.3920455873012543,800.0 -200,300,800,900,0.07861732691526413,11.6358060836792,0.37713393568992615,800.0 -300,400,900,1000,0.08863385021686554,11.35997200012207,0.37710341811180115,800.0 -400,500,1000,1100,0.08568716049194336,11.36007308959961,0.41172850131988525,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 3386436e8f94c1fa3e7eef71c58e918c8616fa89..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.056472789496183395,13.887470245361328,0.26398536562919617,800.0 -100,200,700,800,0.06931658089160919,12.327244758605957,0.3055758774280548,800.0 -200,300,800,900,0.07779020071029663,11.845198631286621,0.3323693871498108,800.0 -300,400,900,1000,0.08062075823545456,11.681880950927734,0.34620240330696106,800.0 -400,500,1000,1100,0.08236729353666306,11.764972686767578,0.3725157082080841,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 23c1dc282a9441514df25159b3b44b28dc363e95..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05678551644086838,13.856801986694336,0.24986809492111206,800.0 -100,200,700,800,0.0679059773683548,12.394038200378418,0.29645827412605286,800.0 -200,300,800,900,0.07764102518558502,12.312858581542969,0.310750275850296,800.0 -300,400,900,1000,0.07851365208625793,12.153739929199219,0.30875587463378906,800.0 -400,500,1000,1100,0.08387082070112228,12.159400939941406,0.3353123366832733,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 51ff808ae54b35691150d2f8548f9715aa6a3583..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.053907360881567,14.263262748718262,0.2405335009098053,800.0 -100,200,700,800,0.06810738146305084,12.566183090209961,0.2986774444580078,800.0 -200,300,800,900,0.07883983850479126,12.010416030883789,0.32240691781044006,800.0 -300,400,900,1000,0.08222272992134094,11.655517578125,0.3202034831047058,800.0 -400,500,1000,1100,0.0868409126996994,11.297654151916504,0.3506008982658386,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index d6dd6602f147d5856c727430a20ca1afc7fb74d0..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.053873833268880844,14.39577865600586,0.2301507145166397,800.0 -100,200,700,800,0.0638396143913269,12.833404541015625,0.2872261106967926,800.0 -200,300,800,900,0.07409082353115082,12.493321418762207,0.3136116862297058,800.0 -300,400,900,1000,0.07632159441709518,12.291511535644531,0.3132152557373047,800.0 -400,500,1000,1100,0.086205393075943,11.530540466308594,0.3491796851158142,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index b5674ed96f786ce7fc254ed636036d835199cee6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05027442425489426,14.999415397644043,0.2164069414138794,800.0 -100,200,700,800,0.059460412710905075,13.293697357177734,0.2672427296638489,800.0 -200,300,800,900,0.06933322548866272,12.747302055358887,0.28716781735420227,800.0 -300,400,900,1000,0.06908018887042999,12.832691192626953,0.2934739589691162,800.0 -400,500,1000,1100,0.08042369782924652,12.057647705078125,0.3330634832382202,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 751db822917d8a76f5f9572c2dc99002e3b6e933..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05103201046586037,14.54443645477295,0.23348356783390045,800.0 -100,200,700,800,0.08803372085094452,11.979907035827637,0.3083576261997223,800.0 -200,300,800,900,0.0842268168926239,12.080499649047852,0.3174397945404053,800.0 -300,400,900,1000,0.08838793635368347,11.68405532836914,0.3365039825439453,800.0 -400,500,1000,1100,0.08472007513046265,11.640384674072266,0.34985053539276123,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index 83f8df89d21d4cbc9609bf03e2caa1fffd38a162..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04085184261202812,15.609489440917969,0.19623765349388123,800.0 -100,200,700,800,0.05202055349946022,13.732282638549805,0.25058209896087646,800.0 -200,300,800,900,0.06277067214250565,13.262367248535156,0.2660787105560303,800.0 -300,400,900,1000,0.05966832488775253,13.470084190368652,0.25990596413612366,800.0 -400,500,1000,1100,0.07314354181289673,12.61832046508789,0.30791816115379333,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index c1d79f77724afdd9822377f8d5b051554a367bfd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04574401676654816,15.447518348693848,0.20486801862716675,800.0 -100,200,700,800,0.0563836395740509,13.766576766967773,0.2507682144641876,800.0 -200,300,800,900,0.06034942716360092,13.506499290466309,0.26658761501312256,800.0 -300,400,900,1000,0.05529975891113281,13.999638557434082,0.24840953946113586,800.0 -400,500,1000,1100,0.07574670761823654,12.598881721496582,0.3294311463832855,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 93860407e659f72c57470625b7efc63a1f9069ec..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04794248566031456,15.441234588623047,0.1989583522081375,800.0 -100,200,700,800,0.05912401154637337,13.490119934082031,0.2537223696708679,800.0 -200,300,800,900,0.06508838385343552,13.119150161743164,0.27013978362083435,800.0 -300,400,900,1000,0.06823047995567322,12.801680564880371,0.29241639375686646,800.0 -400,500,1000,1100,0.06900263577699661,12.790327072143555,0.30882224440574646,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index 74286056a2007df11bdec43acf94ed1db6ad7863..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.039901379495859146,15.688989639282227,0.19968309998512268,800.0 -100,200,700,800,0.05056006461381912,14.058947563171387,0.23804444074630737,800.0 -200,300,800,900,0.05674761161208153,13.450221061706543,0.2571326494216919,800.0 -300,400,900,1000,0.05463392287492752,13.624102592468262,0.25317344069480896,800.0 -400,500,1000,1100,0.06507483124732971,13.037093162536621,0.2990845739841461,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index fe4cb0bed9cb3c4fd18d7151fc5a18d7afc18e1f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03793690353631973,16.24323081970215,0.1827814280986786,800.0 -100,200,700,800,0.051173135638237,14.307661056518555,0.23529480397701263,800.0 -200,300,800,900,0.05536384880542755,13.873748779296875,0.24167987704277039,800.0 -300,400,900,1000,0.05370026454329491,13.99307632446289,0.2325839251279831,800.0 -400,500,1000,1100,0.07637640833854675,12.748442649841309,0.3062785267829895,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 17faa713950d86504adb37ef889d6fc464ffaf3b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0349920317530632,16.3065242767334,0.1788298636674881,800.0 -100,200,700,800,0.0458596907556057,14.839168548583984,0.2231503278017044,800.0 -200,300,800,900,0.057988740503787994,13.902713775634766,0.2488233894109726,800.0 -300,400,900,1000,0.05110115185379982,14.312304496765137,0.24714794754981995,800.0 -400,500,1000,1100,0.06446836143732071,13.209497451782227,0.2967747151851654,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index dff79984947c95502cc43654175b182c53e3be0e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03733573853969574,16.089035034179688,0.18436314165592194,800.0 -100,200,700,800,0.046940457075834274,14.309006690979004,0.23350495100021362,800.0 -200,300,800,900,0.05606872960925102,13.873291015625,0.24669910967350006,800.0 -300,400,900,1000,0.049271754920482635,14.097533226013184,0.23106105625629425,800.0 -400,500,1000,1100,0.06547269225120544,13.004861831665039,0.29471883177757263,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index 4c71a590e3ce7532b71e7e46e3447548c67c1b47..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03460829332470894,16.51884651184082,0.17969027161598206,800.0 -100,200,700,800,0.05227261409163475,14.542682647705078,0.2422412484884262,800.0 -200,300,800,900,0.06520188599824905,13.772735595703125,0.2726680040359497,800.0 -300,400,900,1000,0.054517630487680435,14.34526252746582,0.2443048506975174,800.0 -400,500,1000,1100,0.06955478340387344,12.849154472351074,0.3363204896450043,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index 49781ee4d07f42025a5b1031a605ab0f63f4cda7..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03686417639255524,16.31037139892578,0.18291160464286804,800.0 -100,200,700,800,0.056463178247213364,14.173087120056152,0.26008597016334534,800.0 -200,300,800,900,0.08120749145746231,13.202500343322754,0.28939491510391235,800.0 -300,400,900,1000,0.0901755541563034,12.217219352722168,0.3140803575515747,800.0 -400,500,1000,1100,0.10120333731174469,11.885095596313477,0.38168540596961975,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index 8ff0134d3d2415fdca393c93049235d63b7cc187..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.038684796541929245,16.313295364379883,0.17638342082500458,800.0 -100,200,700,800,0.04217451065778732,14.973986625671387,0.20848685503005981,800.0 -200,300,800,900,0.050192851573228836,14.19937515258789,0.2293483316898346,800.0 -300,400,900,1000,0.05111725255846977,14.280403137207031,0.23673881590366364,800.0 -400,500,1000,1100,0.062217507511377335,13.521095275878906,0.2765146493911743,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 6a6bdebc0dc1c52e343cd14fd08cc98f517470f9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03260737285017967,16.710416793823242,0.16684859991073608,800.0 -100,200,700,800,0.0400509275496006,15.234971046447754,0.20858055353164673,800.0 -200,300,800,900,0.05210559442639351,14.288471221923828,0.2469448447227478,800.0 -300,400,900,1000,0.04911704361438751,14.35607624053955,0.2403002232313156,800.0 -400,500,1000,1100,0.06076136976480484,13.517572402954102,0.2875155508518219,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index d14ccfd0038698c7216d3ab59bfc2b20bbf88763..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.044777680188417435,16.065628051757812,0.29075154662132263,800.0 -100,200,700,800,0.06537003815174103,13.921114921569824,0.3446390628814697,800.0 -200,300,800,900,0.07607249915599823,13.003811836242676,0.3298376202583313,800.0 -300,400,900,1000,0.07717961072921753,12.269582748413086,0.35794487595558167,800.0 -400,500,1000,1100,0.08688884973526001,12.880881309509277,0.37554261088371277,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 6c8b3dc2b36cb405dbc545f9da78218545e74f79..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03472939133644104,20.018211364746094,0.18715709447860718,800.0 -100,200,700,800,0.04438852146267891,17.476001739501953,0.24478818476200104,800.0 -200,300,800,900,0.04765947908163071,19.033605575561523,0.23893621563911438,800.0 -300,400,900,1000,0.07795485109090805,14.181031227111816,0.3226779103279114,800.0 -400,500,1000,1100,0.05540407821536064,16.066015243530273,0.31550467014312744,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index a9c2cf4a9cab8e9aa8039871386b9b7aefbbb145..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.030842935666441917,20.940929412841797,0.1602109968662262,800.0 -100,200,700,800,0.04160571098327637,17.749404907226562,0.21767494082450867,800.0 -200,300,800,900,0.04378768429160118,19.53765106201172,0.23201873898506165,800.0 -300,400,900,1000,0.06081315502524376,15.299939155578613,0.25813964009284973,800.0 -400,500,1000,1100,0.036055222153663635,18.40431785583496,0.23429082334041595,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index bed7d8ef91d57db20595e365b4d70d85de0a63ba..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02451545186340809,20.978374481201172,0.1604473888874054,800.0 -100,200,700,800,0.03571639955043793,18.78231430053711,0.21433116495609283,800.0 -200,300,800,900,0.04146973043680191,19.39521026611328,0.23956231772899628,800.0 -300,400,900,1000,0.06224501132965088,14.68515396118164,0.30187779664993286,800.0 -400,500,1000,1100,0.04019148647785187,16.780139923095703,0.2571616768836975,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index b353f798f78ece3eb341f6af8bb5c8f7920ddd89..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023291578516364098,21.10454559326172,0.15022176504135132,800.0 -100,200,700,800,0.027338000014424324,19.345867156982422,0.18194633722305298,800.0 -200,300,800,900,0.02990860864520073,20.229616165161133,0.19138723611831665,800.0 -300,400,900,1000,0.04117477312684059,15.99724006652832,0.20288974046707153,800.0 -400,500,1000,1100,0.025514107197523117,19.402240753173828,0.1839558482170105,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 54d0bb41f2864e0388e30b4ea29281a75b5d5a82..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025881968438625336,20.54620361328125,0.1650395393371582,800.0 -100,200,700,800,0.042713116854429245,18.307767868041992,0.2215658724308014,800.0 -200,300,800,900,0.04685506969690323,18.818559646606445,0.2501313090324402,800.0 -300,400,900,1000,0.0543840266764164,15.358053207397461,0.2827828526496887,800.0 -400,500,1000,1100,0.039560653269290924,17.70103645324707,0.2610638737678528,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index cbf5dd42d71803e8849882ad6953e5c30fe53841..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026393847540020943,20.769765853881836,0.14386340975761414,800.0 -100,200,700,800,0.06195313483476639,16.53260040283203,0.2329348623752594,800.0 -200,300,800,900,0.04053427651524544,19.321861267089844,0.21171562373638153,800.0 -300,400,900,1000,0.05770866572856903,14.863542556762695,0.2609027028083801,800.0 -400,500,1000,1100,0.06176687404513359,16.559917449951172,0.2635999917984009,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 342cd012e4d0eabd0839a4233b6704744f545aa2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02519858442246914,20.974756240844727,0.14700892567634583,800.0 -100,200,700,800,0.03613204136490822,19.390064239501953,0.1791791021823883,800.0 -200,300,800,900,0.03554737940430641,20.0817813873291,0.20042406022548676,800.0 -300,400,900,1000,0.04464492201805115,15.855582237243652,0.22383634746074677,800.0 -400,500,1000,1100,0.026805240660905838,19.374608993530273,0.2090480625629425,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 6861875262f03f2625abed611b71474bdfbe6368..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024907179176807404,21.090829849243164,0.15352199971675873,800.0 -100,200,700,800,0.030752018094062805,19.170040130615234,0.19407321512699127,800.0 -200,300,800,900,0.03333644941449165,20.28965187072754,0.20570425689220428,800.0 -300,400,900,1000,0.0475120022892952,16.448238372802734,0.2204757183790207,800.0 -400,500,1000,1100,0.02868937887251377,19.31094741821289,0.21539615094661713,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index 5bb02ca591999fab6c549490d8d8442699285720..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03049846924841404,20.350669860839844,0.16857880353927612,800.0 -100,200,700,800,0.06180276721715927,16.760662078857422,0.26935288310050964,800.0 -200,300,800,900,0.08318563550710678,15.572620391845703,0.32276150584220886,800.0 -300,400,900,1000,0.07561735808849335,13.42367935180664,0.34339919686317444,800.0 -400,500,1000,1100,0.05740247294306755,15.991333961486816,0.2970636487007141,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 894cb7e2e112524fe14fd0bdb7d091b44fc26e88..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029576830565929413,20.457279205322266,0.15448413789272308,800.0 -100,200,700,800,0.047989875078201294,17.872426986694336,0.198553204536438,800.0 -200,300,800,900,0.04393831267952919,18.430753707885742,0.21076463162899017,800.0 -300,400,900,1000,0.04340868070721626,16.031002044677734,0.21607670187950134,800.0 -400,500,1000,1100,0.04175862297415733,18.220502853393555,0.22574277222156525,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index fa99b3553c01fc8450458f0fb024c179c1f3a154..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023004136979579926,21.21666717529297,0.15086549520492554,800.0 -100,200,700,800,0.041584596037864685,18.516056060791016,0.20081384479999542,800.0 -200,300,800,900,0.033356569707393646,20.09257698059082,0.20087489485740662,800.0 -300,400,900,1000,0.03690960258245468,16.803607940673828,0.19157946109771729,800.0 -400,500,1000,1100,0.027442974969744682,19.583040237426758,0.20899777114391327,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index a876deaeb1b5a991471e39ba168899173fc64066..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021330051124095917,21.086956024169922,0.14788983762264252,800.0 -100,200,700,800,0.03029903583228588,19.11640739440918,0.18738819658756256,800.0 -200,300,800,900,0.03311878815293312,19.880126953125,0.2065202295780182,800.0 -300,400,900,1000,0.046175021678209305,16.052715301513672,0.2287691831588745,800.0 -400,500,1000,1100,0.028297141194343567,19.025774002075195,0.21359452605247498,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 0602ee1f3d87ca893b81b7ede025c2afe06129b6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023905329406261444,21.002172470092773,0.14064832031726837,800.0 -100,200,700,800,0.03324291482567787,18.739112854003906,0.17985248565673828,800.0 -200,300,800,900,0.04188358411192894,18.585102081298828,0.20516638457775116,800.0 -300,400,900,1000,0.04781736806035042,15.563697814941406,0.21621374785900116,800.0 -400,500,1000,1100,0.029582729563117027,19.03450584411621,0.19568423926830292,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index b1968efafc9d69060b64fd69dfc0e047d15ea568..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020905738696455956,21.193937301635742,0.13468311727046967,800.0 -100,200,700,800,0.042887069284915924,18.15178871154785,0.2055588960647583,800.0 -200,300,800,900,0.04216823726892471,19.619962692260742,0.2209550142288208,800.0 -300,400,900,1000,0.060003429651260376,14.841347694396973,0.2734750807285309,800.0 -400,500,1000,1100,0.05194602534174919,16.702486038208008,0.2648489475250244,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index c640f90acb651740ebac4bf65cb42cc630764182..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025860808789730072,21.34268569946289,0.13993963599205017,800.0 -100,200,700,800,0.031104981899261475,19.321563720703125,0.18224094808101654,800.0 -200,300,800,900,0.07047796994447708,16.947084426879883,0.2466047704219818,800.0 -300,400,900,1000,0.04743868485093117,16.079736709594727,0.21131455898284912,800.0 -400,500,1000,1100,0.05151958391070366,17.601463317871094,0.24787183105945587,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index 80d36b959e8c6353855ce552fe3620aa2de7c787..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01982017792761326,21.489721298217773,0.12934230268001556,800.0 -100,200,700,800,0.04195323586463928,18.918048858642578,0.19625328481197357,800.0 -200,300,800,900,0.052677858620882034,17.652345657348633,0.24545274674892426,800.0 -300,400,900,1000,0.06387761235237122,14.736420631408691,0.3033280670642853,800.0 -400,500,1000,1100,0.04602814093232155,16.944847106933594,0.2767847776412964,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index 38e0d0809e64ef01c33802c769ba591e860a712e..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03808213025331497,20.342321395874023,0.16537265479564667,800.0 -100,200,700,800,0.059443045407533646,17.64259910583496,0.21216458082199097,800.0 -200,300,800,900,0.03602876514196396,18.910966873168945,0.1981383115053177,800.0 -300,400,900,1000,0.05093302205204964,15.606998443603516,0.2492379993200302,800.0 -400,500,1000,1100,0.04093639925122261,18.472640991210938,0.22763580083847046,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 2f53efb95135187983636d35506f73e4807380cc..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022770371288061142,20.81909942626953,0.1445416808128357,800.0 -100,200,700,800,0.04411918297410011,18.23601722717285,0.20512588322162628,800.0 -200,300,800,900,0.04176295921206474,18.75726890563965,0.23165784776210785,800.0 -300,400,900,1000,0.05953088775277138,14.532261848449707,0.27572107315063477,800.0 -400,500,1000,1100,0.04111210256814957,17.43996810913086,0.22898593544960022,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index 5e4798c0b0f36c0cc0857f9514ec084b1befaeb5..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02382185310125351,21.073936462402344,0.134133979678154,800.0 -100,200,700,800,0.038722410798072815,18.505359649658203,0.19239802658557892,800.0 -200,300,800,900,0.04084353893995285,18.399673461914062,0.22795651853084564,800.0 -300,400,900,1000,0.06194061413407326,14.531464576721191,0.28653496503829956,800.0 -400,500,1000,1100,0.04039635509252548,17.586366653442383,0.2537347078323364,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index 952362346b682162c6b3ef2167eb80628f1a1617..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02280047908425331,21.087509155273438,0.1382090002298355,800.0 -100,200,700,800,0.034029871225357056,18.591232299804688,0.17781709134578705,800.0 -200,300,800,900,0.028484122827649117,19.73198699951172,0.1792392134666443,800.0 -300,400,900,1000,0.03633439913392067,16.49799346923828,0.18372052907943726,800.0 -400,500,1000,1100,0.03391486778855324,18.713340759277344,0.20785461366176605,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index f46a1a494cf6067d080ed212e2975d20b8b72876..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021969767287373543,21.250768661499023,0.14425310492515564,800.0 -100,200,700,800,0.04329359531402588,18.850162506103516,0.2026132047176361,800.0 -200,300,800,900,0.041041869670152664,18.349851608276367,0.2183839976787567,800.0 -300,400,900,1000,0.047823525965213776,16.37700653076172,0.2092331498861313,800.0 -400,500,1000,1100,0.02704591304063797,18.907482147216797,0.19406093657016754,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index c8ec18f8bc5254313c30e69e6df1806ab692786a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017969096079468727,21.754131317138672,0.11918800324201584,800.0 -100,200,700,800,0.03904698044061661,18.947118759155273,0.19263151288032532,800.0 -200,300,800,900,0.044901758432388306,17.788959503173828,0.22685126960277557,800.0 -300,400,900,1000,0.04981031268835068,15.176888465881348,0.21919454634189606,800.0 -400,500,1000,1100,0.040255408734083176,16.365707397460938,0.24249042570590973,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index 8e71c82351ada622758c257c234dc56105881e6f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021720057353377342,20.927021026611328,0.14345695078372955,800.0 -100,200,700,800,0.0384686142206192,18.345500946044922,0.2144222855567932,800.0 -200,300,800,900,0.04125683382153511,18.651752471923828,0.2255820482969284,800.0 -300,400,900,1000,0.0477396622300148,15.837011337280273,0.2211267352104187,800.0 -400,500,1000,1100,0.036892689764499664,18.41600227355957,0.22536054253578186,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index 2c69cb1f0786ec3666684f9fbb1ce94cf293ae52..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025168251246213913,20.83905601501465,0.14975784718990326,800.0 -100,200,700,800,0.0346914678812027,18.855937957763672,0.1864510327577591,800.0 -200,300,800,900,0.038331467658281326,18.645240783691406,0.21251140534877777,800.0 -300,400,900,1000,0.047190770506858826,16.131072998046875,0.22263771295547485,800.0 -400,500,1000,1100,0.03469033166766167,17.818368911743164,0.226565882563591,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 25a95508ebe453d0810d8206d786aa82cb987f2a..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025405704975128174,20.242765426635742,0.1553095579147339,800.0 -100,200,700,800,0.030970826745033264,18.96128273010254,0.19526849687099457,800.0 -200,300,800,900,0.0350520983338356,18.972429275512695,0.20862668752670288,800.0 -300,400,900,1000,0.04531605541706085,15.637259483337402,0.22142040729522705,800.0 -400,500,1000,1100,0.02643098682165146,19.231191635131836,0.2067740559577942,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 24b508597e6df9f6b72dd3d9229193a4c41b2dfa..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020186960697174072,21.424299240112305,0.12936578691005707,800.0 -100,200,700,800,0.034036800265312195,19.347179412841797,0.1740259975194931,800.0 -200,300,800,900,0.05015106126666069,17.92537498474121,0.2188211977481842,800.0 -300,400,900,1000,0.046468451619148254,16.030773162841797,0.19965483248233795,800.0 -400,500,1000,1100,0.02887565828859806,19.311975479125977,0.1967252492904663,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 3d0722ef41ccec6cdc784f5f646932f59c9ebfd8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01873735338449478,21.521018981933594,0.12085296958684921,800.0 -100,200,700,800,0.03158377856016159,19.129493713378906,0.1802210658788681,800.0 -200,300,800,900,0.03430667519569397,19.382177352905273,0.20496580004692078,800.0 -300,400,900,1000,0.03906508535146713,16.046323776245117,0.19198505580425262,800.0 -400,500,1000,1100,0.023190679028630257,19.797643661499023,0.19288241863250732,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index d2b806b44b0909f8fbe952451222175b7358436f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018467778339982033,21.648998260498047,0.1242322325706482,800.0 -100,200,700,800,0.03777126595377922,19.24671745300293,0.1781201958656311,800.0 -200,300,800,900,0.035844773054122925,18.879377365112305,0.20046691596508026,800.0 -300,400,900,1000,0.04745113477110863,16.491540908813477,0.20981480181217194,800.0 -400,500,1000,1100,0.028486089780926704,19.422313690185547,0.20307207107543945,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index 50d058d9e15d36ceffb094366575b919627304df..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018993282690644264,21.419879913330078,0.1206824854016304,800.0 -100,200,700,800,0.039762601256370544,18.639476776123047,0.18817944824695587,800.0 -200,300,800,900,0.03814339265227318,19.010385513305664,0.20276473462581635,800.0 -300,400,900,1000,0.04482186958193779,16.386953353881836,0.2117519974708557,800.0 -400,500,1000,1100,0.029484529048204422,18.854328155517578,0.1948135495185852,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index 739f0a967119ad99c0a0c378432f36710d4d3277..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024757228791713715,20.689603805541992,0.1357138305902481,800.0 -100,200,700,800,0.04249553754925728,18.103914260864258,0.189509779214859,800.0 -200,300,800,900,0.04978874325752258,18.086889266967773,0.2212337851524353,800.0 -300,400,900,1000,0.050519876182079315,15.99595832824707,0.2188204526901245,800.0 -400,500,1000,1100,0.03234812244772911,18.31662368774414,0.2117481678724289,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 55b69eb3dc5071fe27c2cea8e6125f24304ad361..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019832078367471695,20.984949111938477,0.13385076820850372,800.0 -100,200,700,800,0.02778242528438568,19.277446746826172,0.1801549196243286,800.0 -200,300,800,900,0.038011182099580765,18.615625381469727,0.2241731882095337,800.0 -300,400,900,1000,0.04639282077550888,16.059375762939453,0.22673456370830536,800.0 -400,500,1000,1100,0.0315144807100296,18.699323654174805,0.23279979825019836,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index b1389daf2ccdb902946407e987b9045c5afb4809..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018319493159651756,21.567502975463867,0.11495664715766907,800.0 -100,200,700,800,0.03843316435813904,18.105602264404297,0.18573638796806335,800.0 -200,300,800,900,0.0432315431535244,18.73456573486328,0.21602462232112885,800.0 -300,400,900,1000,0.051058005541563034,16.549678802490234,0.2066759169101715,800.0 -400,500,1000,1100,0.053147658705711365,18.52375030517578,0.22481052577495575,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index febf9aec84325da7f3695f403edc983ce49aa493..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018963105976581573,21.614152908325195,0.11861380934715271,800.0 -100,200,700,800,0.04200465604662895,18.330425262451172,0.1945715844631195,800.0 -200,300,800,900,0.044299740344285965,18.472970962524414,0.2348983734846115,800.0 -300,400,900,1000,0.05632099509239197,15.623383522033691,0.2559983730316162,800.0 -400,500,1000,1100,0.032715462148189545,18.335901260375977,0.22938844561576843,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index fd440b5b8dae4a4569d30b566527bb939362f2d0..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0254471804946661,20.438766479492188,0.1649637222290039,800.0 -100,200,700,800,0.03211626037955284,18.802474975585938,0.19867658615112305,800.0 -200,300,800,900,0.040054820477962494,18.061565399169922,0.24851976335048676,800.0 -300,400,900,1000,0.0584409199655056,14.499347686767578,0.25931718945503235,800.0 -400,500,1000,1100,0.051106058061122894,17.250139236450195,0.2628611624240875,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 831a6eea85dfe6092b92b04286e1f848f62335e9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015582325868308544,22.286828994750977,0.1012340560555458,800.0 -100,200,700,800,0.02560894936323166,19.78899383544922,0.160252183675766,800.0 -200,300,800,900,0.029106728732585907,19.380661010742188,0.19725704193115234,800.0 -300,400,900,1000,0.04099899157881737,16.43575668334961,0.22041015326976776,800.0 -400,500,1000,1100,0.03333475440740585,18.669485092163086,0.20858639478683472,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index 80d9708457dfea82fdd1e8fb3f9b5c427a9f2bc9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023957224562764168,20.557598114013672,0.1364031434059143,800.0 -100,200,700,800,0.04979578033089638,17.738082885742188,0.2059425711631775,800.0 -200,300,800,900,0.043720588088035583,18.5267333984375,0.2191716730594635,800.0 -300,400,900,1000,0.05318243056535721,16.019567489624023,0.2021520584821701,800.0 -400,500,1000,1100,0.07455144077539444,16.114797592163086,0.2481677085161209,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 463dc77b8a346b466a3d53c2f196f6ae4d4deb65..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01829340308904648,21.738698959350586,0.12045799940824509,800.0 -100,200,700,800,0.04100818186998367,18.009464263916016,0.20315046608448029,800.0 -200,300,800,900,0.048741087317466736,17.585857391357422,0.23341387510299683,800.0 -300,400,900,1000,0.05582535266876221,15.382473945617676,0.25652092695236206,800.0 -400,500,1000,1100,0.03929414600133896,17.310741424560547,0.23273950815200806,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 7393923ee5d6dfd7d1750641af0d0b4f6ee30639..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01635589636862278,22.021589279174805,0.10964234173297882,800.0 -100,200,700,800,0.03573949635028839,18.77216148376465,0.18268722295761108,800.0 -200,300,800,900,0.03887684643268585,18.617101669311523,0.22221584618091583,800.0 -300,400,900,1000,0.05985463783144951,15.127289772033691,0.25485432147979736,800.0 -400,500,1000,1100,0.05393361672759056,17.860801696777344,0.253678560256958,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 61638257f0cda247619e7093af992ed00ec57290..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021678099408745766,21.126943588256836,0.13322056829929352,800.0 -100,200,700,800,0.028962915763258934,19.224327087402344,0.17805948853492737,800.0 -200,300,800,900,0.03096286952495575,19.60422706604004,0.19058440625667572,800.0 -300,400,900,1000,0.0382290855050087,16.925363540649414,0.18134494125843048,800.0 -400,500,1000,1100,0.023089777678251266,20.153423309326172,0.18363045156002045,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index e5e2f373b7cbbb647672748b1ce261e94a6693ed..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021544329822063446,21.081989288330078,0.1416759341955185,800.0 -100,200,700,800,0.03109770454466343,18.79755210876465,0.18709491193294525,800.0 -200,300,800,900,0.03658674284815788,18.503591537475586,0.20899119973182678,800.0 -300,400,900,1000,0.03311138227581978,16.956029891967773,0.1734803020954132,800.0 -400,500,1000,1100,0.033863648772239685,18.3748722076416,0.2109437882900238,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index 1ed76e744e7dae01dfc61fd743e6adf39d0c282b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02062683179974556,21.193605422973633,0.1440756916999817,800.0 -100,200,700,800,0.045352596789598465,17.418912887573242,0.21803288161754608,800.0 -200,300,800,900,0.04403974488377571,17.611352920532227,0.229207843542099,800.0 -300,400,900,1000,0.061339594423770905,14.74726676940918,0.25853124260902405,800.0 -400,500,1000,1100,0.053112417459487915,15.968527793884277,0.2492469847202301,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index abcf65c830482c52f548502b6dbb2cbd5290cd09..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019320795312523842,21.30278968811035,0.13092267513275146,800.0 -100,200,700,800,0.03952318802475929,18.797927856445312,0.18998880684375763,800.0 -200,300,800,900,0.038124989718198776,18.879281997680664,0.21458224952220917,800.0 -300,400,900,1000,0.044964369386434555,15.871782302856445,0.21061736345291138,800.0 -400,500,1000,1100,0.05157533288002014,17.378154754638672,0.24114371836185455,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index 447c81f11972046384551bfc7cfe1a4ab27aa729..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01709684170782566,21.618824005126953,0.11839044839143753,800.0 -100,200,700,800,0.042853862047195435,18.305530548095703,0.1829880326986313,800.0 -200,300,800,900,0.03826172277331352,18.815492630004883,0.20525436103343964,800.0 -300,400,900,1000,0.04288482666015625,16.28145408630371,0.19074462354183197,800.0 -400,500,1000,1100,0.04814969003200531,17.082704544067383,0.23203207552433014,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index 9b73727cd53b8f8f1550e5ba51f28e381066da32..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01719527877867222,21.951622009277344,0.11152631044387817,800.0 -100,200,700,800,0.04173271358013153,18.955856323242188,0.17247432470321655,800.0 -200,300,800,900,0.06112814322113991,17.782686233520508,0.21340174973011017,800.0 -300,400,900,1000,0.04404857009649277,16.936933517456055,0.17597055435180664,800.0 -400,500,1000,1100,0.025449300184845924,19.514467239379883,0.17322193086147308,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index fe9ea6ec66cb7cbafba1c6c59723e6384e81fec9..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016945524141192436,21.318286895751953,0.12549452483654022,800.0 -100,200,700,800,0.03641045093536377,18.623205184936523,0.18996348977088928,800.0 -200,300,800,900,0.037344783544540405,19.26616859436035,0.19739173352718353,800.0 -300,400,900,1000,0.03386810049414635,17.204879760742188,0.1897551566362381,800.0 -400,500,1000,1100,0.026649635285139084,19.469642639160156,0.19766363501548767,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index 272b7d2c66863c9563ec23b00712c1c2d8eb8e41..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020167088136076927,21.496278762817383,0.11604885756969452,800.0 -100,200,700,800,0.03267623484134674,18.788284301757812,0.18199485540390015,800.0 -200,300,800,900,0.037417374551296234,18.878501892089844,0.21418353915214539,800.0 -300,400,900,1000,0.033523477613925934,17.149612426757812,0.17763157188892365,800.0 -400,500,1000,1100,0.02464284747838974,19.698572158813477,0.19419145584106445,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index ee72dcec0299dca05f1791307bf4633e546b0cec..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018338643014431,21.796289443969727,0.11608345061540604,800.0 -100,200,700,800,0.029994197189807892,19.27613067626953,0.1729494035243988,800.0 -200,300,800,900,0.03240946680307388,19.327220916748047,0.2004110962152481,800.0 -300,400,900,1000,0.043769195675849915,16.75751495361328,0.20619401335716248,800.0 -400,500,1000,1100,0.02718341164290905,19.501924514770508,0.2052253782749176,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index 43a14110a1e319745831231d03d18994246884ce..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01879964768886566,21.59717559814453,0.12211019545793533,800.0 -100,200,700,800,0.03861869499087334,18.481002807617188,0.1924617439508438,800.0 -200,300,800,900,0.03577577695250511,18.774555206298828,0.19478724896907806,800.0 -300,400,900,1000,0.035171717405319214,16.834749221801758,0.18203361332416534,800.0 -400,500,1000,1100,0.024915998801589012,19.571739196777344,0.19285108149051666,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index f70187c1d10ec87c91718991ac01eec46c6ddd23..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018943052738904953,21.621187210083008,0.11812878400087357,800.0 -100,200,700,800,0.03664608299732208,19.208961486816406,0.16965757310390472,800.0 -200,300,800,900,0.040644582360982895,18.9277400970459,0.2086053490638733,800.0 -300,400,900,1000,0.04170724377036095,16.47930145263672,0.21017950773239136,800.0 -400,500,1000,1100,0.041743699461221695,17.578994750976562,0.22216545045375824,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 6d9f077a9b196783947e3a7a76f3ae0678d0bea6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015704667195677757,21.93885040283203,0.11667688190937042,800.0 -100,200,700,800,0.026373762637376785,19.382293701171875,0.17070531845092773,800.0 -200,300,800,900,0.03446795046329498,19.387178421020508,0.2062869668006897,800.0 -300,400,900,1000,0.03789421170949936,16.31731605529785,0.19408626854419708,800.0 -400,500,1000,1100,0.036198992282152176,19.297258377075195,0.19974933564662933,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index e6910dc9c0e5a4dfaf23b02d91fec9e683d661c2..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017651483416557312,21.91132354736328,0.11343774199485779,800.0 -100,200,700,800,0.03628063574433327,19.469335556030273,0.17959511280059814,800.0 -200,300,800,900,0.037498537451028824,19.184114456176758,0.21893846988677979,800.0 -300,400,900,1000,0.04376070201396942,16.799333572387695,0.20445027947425842,800.0 -400,500,1000,1100,0.02536226063966751,19.667186737060547,0.20841719210147858,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index 517592da02311a4ed322ce24fa6c9955d35d09d6..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02141096629202366,21.289325714111328,0.1210077777504921,800.0 -100,200,700,800,0.03762727603316307,18.633012771606445,0.17605724930763245,800.0 -200,300,800,900,0.03185776621103287,19.49662208557129,0.18366119265556335,800.0 -300,400,900,1000,0.04452693834900856,16.28316879272461,0.20642554759979248,800.0 -400,500,1000,1100,0.04166027158498764,18.286439895629883,0.20844194293022156,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index 72f537e6db1d2ab214e8610a106ffb87206e0848..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019553590565919876,21.511369705200195,0.12245352566242218,800.0 -100,200,700,800,0.037630271166563034,18.810157775878906,0.1756298989057541,800.0 -200,300,800,900,0.03699730709195137,18.52789878845215,0.20131704211235046,800.0 -300,400,900,1000,0.04697752371430397,16.185155868530273,0.19965864717960358,800.0 -400,500,1000,1100,0.04077695310115814,17.45354461669922,0.22294214367866516,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index cab6727fedb1c44f9ae065aa3e4f72abf5969989..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020252995193004608,21.37453842163086,0.12555550038814545,800.0 -100,200,700,800,0.04455868899822235,18.039836883544922,0.20231957733631134,800.0 -200,300,800,900,0.042896758764982224,18.30759620666504,0.21559150516986847,800.0 -300,400,900,1000,0.04757942259311676,15.842368125915527,0.2090734988451004,800.0 -400,500,1000,1100,0.03344714269042015,18.848108291625977,0.2092798799276352,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index 57e22d98b1aff108c97a2b26df31bae4aa12cc0b..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02276964671909809,21.3739070892334,0.12847958505153656,800.0 -100,200,700,800,0.04169367626309395,18.486841201782227,0.20150379836559296,800.0 -200,300,800,900,0.057631783187389374,18.07455825805664,0.23890462517738342,800.0 -300,400,900,1000,0.049751728773117065,16.115001678466797,0.21301834285259247,800.0 -400,500,1000,1100,0.037737008184194565,17.965190887451172,0.23445004224777222,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00152000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00152000.csv deleted file mode 100644 index b4aa48aed8c5d141f8222a18434ccf6036c1c164..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00152000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017965765669941902,21.774938583374023,0.1301557868719101,800.0 -100,200,700,800,0.04125094413757324,19.432825088500977,0.18868690729141235,800.0 -200,300,800,900,0.0373435914516449,19.51922035217285,0.20634284615516663,800.0 -300,400,900,1000,0.045576829463243484,16.708349227905273,0.20946469902992249,800.0 -400,500,1000,1100,0.029531991109251976,19.320751190185547,0.2186640202999115,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00154000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00154000.csv deleted file mode 100644 index be1ec63967c2d42fca0f7db5f18709b8fb709877..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00154000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02110007219016552,21.272987365722656,0.14898768067359924,800.0 -100,200,700,800,0.0449814572930336,18.233661651611328,0.2041136771440506,800.0 -200,300,800,900,0.032178159803152084,19.820833206176758,0.20382413268089294,800.0 -300,400,900,1000,0.03976937383413315,16.68965721130371,0.19975171983242035,800.0 -400,500,1000,1100,0.04067721962928772,17.54663848876953,0.2376721203327179,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00156000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00156000.csv deleted file mode 100644 index 108663032c160435b2a0a532f27dc2989b1f7dd8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00156000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017359068617224693,21.857717514038086,0.11439258605241776,800.0 -100,200,700,800,0.03063312917947769,19.323501586914062,0.18671855330467224,800.0 -200,300,800,900,0.04205116629600525,18.5189266204834,0.24107937514781952,800.0 -300,400,900,1000,0.05333949998021126,16.239608764648438,0.23833255469799042,800.0 -400,500,1000,1100,0.04313385859131813,17.798219680786133,0.2466421276330948,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00158000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00158000.csv deleted file mode 100644 index 3c0bae9f3b9dd5d5bedefb2e17ba4328889ac56c..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00158000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019820254296064377,21.2672061920166,0.1365271508693695,800.0 -100,200,700,800,0.028807537630200386,19.684635162353516,0.17261403799057007,800.0 -200,300,800,900,0.04587365314364433,19.448593139648438,0.19047246873378754,800.0 -300,400,900,1000,0.038322459906339645,16.941570281982422,0.18115463852882385,800.0 -400,500,1000,1100,0.03264729678630829,18.362289428710938,0.21066229045391083,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00160000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00160000.csv deleted file mode 100644 index 532d5b4840cd27ca95b17d046ac7ffbe39f188c4..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00160000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018721850588917732,21.448671340942383,0.15028806030750275,800.0 -100,200,700,800,0.03699963167309761,18.983022689819336,0.19731849431991577,800.0 -200,300,800,900,0.03555301949381828,19.075546264648438,0.20508815348148346,800.0 -300,400,900,1000,0.051769960671663284,15.315340995788574,0.2359006255865097,800.0 -400,500,1000,1100,0.027340807020664215,19.250473022460938,0.21210883557796478,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00162000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00162000.csv deleted file mode 100644 index ed3b66f9765dcee7412a74dad9b0bb978c5a610f..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00162000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017215032130479813,21.565771102905273,0.12459584325551987,800.0 -100,200,700,800,0.029648328199982643,19.445842742919922,0.17867140471935272,800.0 -200,300,800,900,0.03543999791145325,19.359540939331055,0.20863337814807892,800.0 -300,400,900,1000,0.031233038753271103,17.5759334564209,0.16618706285953522,800.0 -400,500,1000,1100,0.029929662123322487,18.828907012939453,0.19137915968894958,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00164000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00164000.csv deleted file mode 100644 index c738400c64dcb15f243c52f5d50b1c28bbaa9ac3..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00164000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01778917945921421,21.813026428222656,0.12341427057981491,800.0 -100,200,700,800,0.03548809140920639,18.870616912841797,0.19305188953876495,800.0 -200,300,800,900,0.03960243612527847,18.64373779296875,0.22622758150100708,800.0 -300,400,900,1000,0.043456487357616425,16.027446746826172,0.23524929583072662,800.0 -400,500,1000,1100,0.03833121061325073,17.679615020751953,0.27079445123672485,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00166000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00166000.csv deleted file mode 100644 index d8311f2d9aa26dd5762522994d353118661b928d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00166000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017730046063661575,21.891860961914062,0.1119011715054512,800.0 -100,200,700,800,0.04050297662615776,18.854604721069336,0.1820542961359024,800.0 -200,300,800,900,0.03461179882287979,19.35934829711914,0.19392409920692444,800.0 -300,400,900,1000,0.030683431774377823,17.39647102355957,0.1649044007062912,800.0 -400,500,1000,1100,0.031228167936205864,19.12470054626465,0.18996958434581757,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00168000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00168000.csv deleted file mode 100644 index f2ca4f48c553cc3567b1a554d57aecf0d873b5ed..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00168000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0178393442183733,21.762887954711914,0.12752144038677216,800.0 -100,200,700,800,0.03824472799897194,19.557649612426758,0.18733897805213928,800.0 -200,300,800,900,0.038310062140226364,18.752412796020508,0.21154484152793884,800.0 -300,400,900,1000,0.034209031611680984,17.033527374267578,0.18133021891117096,800.0 -400,500,1000,1100,0.027652500197291374,18.974313735961914,0.1980881690979004,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00170000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00170000.csv deleted file mode 100644 index fc7a3d2dfcb10e5a7e80eaac98a61de57e886783..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00170000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017348188906908035,21.69989013671875,0.11804032325744629,800.0 -100,200,700,800,0.03587007150053978,18.507474899291992,0.19424739480018616,800.0 -200,300,800,900,0.05536934360861778,17.657617568969727,0.25086915493011475,800.0 -300,400,900,1000,0.05994589626789093,15.220759391784668,0.2582249045372009,800.0 -400,500,1000,1100,0.05976765975356102,15.90609359741211,0.2737063467502594,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00172000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00172000.csv deleted file mode 100644 index 14df3f594074308f3f6874100fee3bc2777bb361..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00172000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016261864453554153,21.776674270629883,0.11769740283489227,800.0 -100,200,700,800,0.03657695651054382,19.50368881225586,0.15442165732383728,800.0 -200,300,800,900,0.04595981910824776,18.381080627441406,0.19461888074874878,800.0 -300,400,900,1000,0.043208323419094086,16.627586364746094,0.20388098061084747,800.0 -400,500,1000,1100,0.026355531066656113,19.232519149780273,0.19239693880081177,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00174000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00174000.csv deleted file mode 100644 index 2e146356d64ac3a813d1835022875217673f4413..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00174000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019260136410593987,21.314821243286133,0.12751758098602295,800.0 -100,200,700,800,0.0456046424806118,18.144615173339844,0.2025476098060608,800.0 -200,300,800,900,0.03183365240693092,19.381763458251953,0.21023987233638763,800.0 -300,400,900,1000,0.03934372961521149,16.887123107910156,0.20682674646377563,800.0 -400,500,1000,1100,0.02896539680659771,18.811073303222656,0.19881533086299896,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00176000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00176000.csv deleted file mode 100644 index 3ef7211c620b0c1dbb24e66a698d41b73145f645..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00176000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01730762980878353,22.060026168823242,0.11637221276760101,800.0 -100,200,700,800,0.035587016493082047,18.615808486938477,0.19922763109207153,800.0 -200,300,800,900,0.03444491699337959,18.62847900390625,0.2240017056465149,800.0 -300,400,900,1000,0.061051979660987854,14.684405326843262,0.28548094630241394,800.0 -400,500,1000,1100,0.0342787429690361,18.397600173950195,0.25065624713897705,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00178000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00178000.csv deleted file mode 100644 index edf03ca5ebc82bccb83c8077d102e33d698ce0ee..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00178000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01335590798407793,22.67538833618164,0.08912909775972366,800.0 -100,200,700,800,0.030516881495714188,19.640287399291992,0.16110549867153168,800.0 -200,300,800,900,0.03653223440051079,19.157943725585938,0.18759259581565857,800.0 -300,400,900,1000,0.04194474220275879,16.399499893188477,0.1995941698551178,800.0 -400,500,1000,1100,0.033831268548965454,18.393117904663086,0.20893919467926025,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00180000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00180000.csv deleted file mode 100644 index bffb38f87bc2b2304bab13d243fcf23fc521f593..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00180000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0175344105809927,21.047983169555664,0.1196913942694664,800.0 -100,200,700,800,0.040067072957754135,18.30489730834961,0.1904793381690979,800.0 -200,300,800,900,0.04390779137611389,19.2038631439209,0.21063636243343353,800.0 -300,400,900,1000,0.0313924178481102,17.457109451293945,0.17164672911167145,800.0 -400,500,1000,1100,0.025383789092302322,19.460594177246094,0.19406552612781525,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00182000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00182000.csv deleted file mode 100644 index be18d063a98fc1106a3fc1b7cff4483a17387194..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00182000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02472197823226452,20.760351181030273,0.1516944169998169,800.0 -100,200,700,800,0.045056309551000595,18.21355438232422,0.20090003311634064,800.0 -200,300,800,900,0.05704834312200546,17.970989227294922,0.23108330368995667,800.0 -300,400,900,1000,0.04956354573369026,15.626415252685547,0.22391164302825928,800.0 -400,500,1000,1100,0.03996630012989044,17.709915161132812,0.22121873497962952,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00184000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00184000.csv deleted file mode 100644 index 0184baa7bac6af8a75f6f0a0eaa49b54ae979c74..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00184000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015100398100912571,22.219470977783203,0.1018514558672905,800.0 -100,200,700,800,0.033469099551439285,19.404747009277344,0.177485391497612,800.0 -200,300,800,900,0.03842882066965103,19.44086265563965,0.20142601430416107,800.0 -300,400,900,1000,0.027877282351255417,17.920663833618164,0.15726590156555176,800.0 -400,500,1000,1100,0.024043474346399307,19.396411895751953,0.18773071467876434,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00186000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00186000.csv deleted file mode 100644 index 60cb646406ee36190467cbcb8750be2213154df8..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00186000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017156384885311127,22.063047409057617,0.1138201504945755,800.0 -100,200,700,800,0.03228829428553581,19.237966537475586,0.16874559223651886,800.0 -200,300,800,900,0.03300219029188156,19.47800064086914,0.19370044767856598,800.0 -300,400,900,1000,0.03529057651758194,17.0127010345459,0.19811780750751495,800.0 -400,500,1000,1100,0.03000691719353199,18.547588348388672,0.22032485902309418,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00188000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00188000.csv deleted file mode 100644 index abf7162887c837baf9b8c0d555f09eff81706bfe..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00188000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.013503320515155792,22.612451553344727,0.0984652116894722,800.0 -100,200,700,800,0.028087936341762543,19.847488403320312,0.1493755429983139,800.0 -200,300,800,900,0.03081403486430645,19.676965713500977,0.17430919408798218,800.0 -300,400,900,1000,0.037086088210344315,17.222064971923828,0.17670947313308716,800.0 -400,500,1000,1100,0.02556740492582321,18.96129035949707,0.1935606598854065,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00190000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00190000.csv deleted file mode 100644 index 712477fd49b071d883d0ebc661a115808dfedffd..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00190000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01979437656700611,21.60742950439453,0.12601640820503235,800.0 -100,200,700,800,0.035174205899238586,18.664352416992188,0.1860356479883194,800.0 -200,300,800,900,0.04660084843635559,18.298168182373047,0.21220391988754272,800.0 -300,400,900,1000,0.03575970232486725,17.19634437561035,0.1893799602985382,800.0 -400,500,1000,1100,0.02820771187543869,18.80015754699707,0.20308104157447815,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00192000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00192000.csv deleted file mode 100644 index 2959b805dffa3e63df878e6933354e75aa78f049..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00192000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01763189770281315,22.21678924560547,0.11263580620288849,800.0 -100,200,700,800,0.03507860004901886,19.455259323120117,0.17969940602779388,800.0 -200,300,800,900,0.0340007059276104,19.51249885559082,0.20548970997333527,800.0 -300,400,900,1000,0.04108956456184387,16.882797241210938,0.2068769484758377,800.0 -400,500,1000,1100,0.024739889428019524,19.76593589782715,0.21760354936122894,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00194000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00194000.csv deleted file mode 100644 index 784b9e7ce546b94e0490c9495658a9cd6ee99622..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00194000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02083495445549488,21.305631637573242,0.13502347469329834,800.0 -100,200,700,800,0.03981392830610275,19.354543685913086,0.19382043182849884,800.0 -200,300,800,900,0.04152015596628189,18.930620193481445,0.23813682794570923,800.0 -300,400,900,1000,0.05266084522008896,16.17139434814453,0.2503339350223541,800.0 -400,500,1000,1100,0.035221122205257416,18.98352813720703,0.23680126667022705,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00196000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00196000.csv deleted file mode 100644 index 87c8c6ebb34b5506abd9b13a25405e9eca74946d..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00196000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017650671303272247,21.877273559570312,0.11776083707809448,800.0 -100,200,700,800,0.03459122031927109,18.790443420410156,0.2095831036567688,800.0 -200,300,800,900,0.06890139728784561,15.839137077331543,0.2921079099178314,800.0 -300,400,900,1000,0.08790630102157593,13.955901145935059,0.3209567368030548,800.0 -400,500,1000,1100,0.09623898565769196,14.017712593078613,0.328377366065979,800.0 diff --git a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00198000.csv b/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00198000.csv deleted file mode 100644 index 44609a15bfef6c78d342d793027915883dc57104..0000000000000000000000000000000000000000 --- a/train_dememwm_sourcefree_ref_8h200_bs16_global128_seed42_200k/wandb/offline-run-20260718_004720-lm8civu6/files/metrics/segment_metrics_step00198000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019740939140319824,21.149202346801758,0.13663385808467865,800.0 -100,200,700,800,0.03223362937569618,19.70225715637207,0.16197089850902557,800.0 -200,300,800,900,0.04270176962018013,19.018051147460938,0.22384628653526306,800.0 -300,400,900,1000,0.03418086841702461,17.237520217895508,0.17866718769073486,800.0 -400,500,1000,1100,0.025560464709997177,19.535200119018555,0.18952365219593048,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index 30565abd121250dff3f061dc02f9c3b592d08ed1..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07078587263822556,12.678794860839844,0.32841596007347107,800.0 -100,200,700,800,0.09517151862382889,10.680359840393066,0.4092690944671631,800.0 -200,300,800,900,0.09007565677165985,11.115741729736328,0.4079235792160034,800.0 -300,400,900,1000,0.08519289642572403,11.256317138671875,0.4177609384059906,800.0 -400,500,1000,1100,0.09876220673322678,10.670623779296875,0.4595567286014557,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index 19c94cea09c4616fb982b6609a100d6757a60225..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06318002939224243,13.271437644958496,0.29223281145095825,800.0 -100,200,700,800,0.0834902673959732,11.337180137634277,0.3717069923877716,800.0 -200,300,800,900,0.09357446432113647,10.701672554016113,0.40301162004470825,800.0 -300,400,900,1000,0.08974650502204895,11.027393341064453,0.38626158237457275,800.0 -400,500,1000,1100,0.09101159870624542,10.879385948181152,0.42727065086364746,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index 5aed968f2405228d9c0efea6bc5b438b0a0fee05..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06839294731616974,12.895771026611328,0.2988506853580475,800.0 -100,200,700,800,0.07543002814054489,11.71527099609375,0.335946649312973,800.0 -200,300,800,900,0.09578661620616913,10.876614570617676,0.37356582283973694,800.0 -300,400,900,1000,0.1007874608039856,10.6320161819458,0.39154052734375,800.0 -400,500,1000,1100,0.11271770298480988,10.114021301269531,0.4214378297328949,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 06fa883745d3905a6ac3d3a90e6136548a001c45..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06002971529960632,13.644331932067871,0.26867714524269104,800.0 -100,200,700,800,0.07217607647180557,12.198573112487793,0.31083205342292786,800.0 -200,300,800,900,0.08889692276716232,11.464734077453613,0.3429679870605469,800.0 -300,400,900,1000,0.07853052020072937,12.201889991760254,0.3119564950466156,800.0 -400,500,1000,1100,0.08156795799732208,11.649971008300781,0.3585025668144226,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index 6625f795aa3d399a1b0e1c5a3c41947c5d39829f..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05164013057947159,14.059375762939453,0.24759721755981445,800.0 -100,200,700,800,0.06799560785293579,12.446986198425293,0.28960129618644714,800.0 -200,300,800,900,0.07545848935842514,12.117851257324219,0.30479636788368225,800.0 -300,400,900,1000,0.07231004536151886,12.426448822021484,0.3042861819267273,800.0 -400,500,1000,1100,0.07971535623073578,12.055516242980957,0.3489055633544922,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 0d5d45c359445d23014328d892337413bbdb3752..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0632801279425621,13.736127853393555,0.26676881313323975,800.0 -100,200,700,800,0.08532102406024933,12.011199951171875,0.34066954255104065,800.0 -200,300,800,900,0.09779428690671921,10.860315322875977,0.3765782117843628,800.0 -300,400,900,1000,0.08397743850946426,11.584023475646973,0.34703072905540466,800.0 -400,500,1000,1100,0.09594736248254776,11.23757553100586,0.40170738101005554,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index 7c062b299ae1f8b7f2451203a6d80753c42f1c59..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.051613327115774155,14.454694747924805,0.2332802563905716,800.0 -100,200,700,800,0.06452997773885727,13.150232315063477,0.28574612736701965,800.0 -200,300,800,900,0.07394594699144363,12.381256103515625,0.3181965947151184,800.0 -300,400,900,1000,0.08641123026609421,11.817852020263672,0.3354194760322571,800.0 -400,500,1000,1100,0.08958129584789276,11.4771089553833,0.37201812863349915,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 9dfa04e1a19aa1958339e3611b4a6b2f1d3715fa..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06106087565422058,13.742351531982422,0.2520599663257599,800.0 -100,200,700,800,0.08195710182189941,11.933158874511719,0.3191249966621399,800.0 -200,300,800,900,0.08547373116016388,11.592411994934082,0.32380545139312744,800.0 -300,400,900,1000,0.0932205468416214,11.389820098876953,0.31294846534729004,800.0 -400,500,1000,1100,0.10039613395929337,11.233184814453125,0.35498368740081787,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index 9b7eaf0db0c428b44d628f4943e028ac0140a573..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.047974079847335815,15.03744125366211,0.2137433886528015,800.0 -100,200,700,800,0.06086605414748192,13.166130065917969,0.26498955488204956,800.0 -200,300,800,900,0.06996271759271622,12.661884307861328,0.28920114040374756,800.0 -300,400,900,1000,0.060670431703329086,13.257152557373047,0.2645406424999237,800.0 -400,500,1000,1100,0.07595790177583694,12.500502586364746,0.3239583671092987,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index ea3785c426e52f2f5af100871e85a568bc755d9c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04813078045845032,14.978893280029297,0.21694734692573547,800.0 -100,200,700,800,0.062665194272995,13.066694259643555,0.2797624170780182,800.0 -200,300,800,900,0.06480882316827774,13.074891090393066,0.2764759063720703,800.0 -300,400,900,1000,0.06800565868616104,12.686662673950195,0.29783982038497925,800.0 -400,500,1000,1100,0.07563749700784683,12.314411163330078,0.3551326096057892,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 36bd40a5db851439483765b5a8c0f99ee69bf8a8..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05075892060995102,14.484155654907227,0.23301178216934204,800.0 -100,200,700,800,0.05976174399256706,12.930118560791016,0.2869741916656494,800.0 -200,300,800,900,0.060377076268196106,13.170623779296875,0.2710047960281372,800.0 -300,400,900,1000,0.05888129398226738,13.187629699707031,0.285653293132782,800.0 -400,500,1000,1100,0.07312040776014328,12.353446960449219,0.3469061255455017,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index fe9ef5db1fd1081f7a7ff65ffcddb0ffd72c36bc..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04128601402044296,15.627257347106934,0.19557422399520874,800.0 -100,200,700,800,0.05180984362959862,13.934399604797363,0.24194084107875824,800.0 -200,300,800,900,0.05904025211930275,13.698713302612305,0.2529850900173187,800.0 -300,400,900,1000,0.0615682452917099,13.528603553771973,0.26468169689178467,800.0 -400,500,1000,1100,0.07227061688899994,12.779775619506836,0.30730578303337097,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index fe006b770149132f11cb83a8508bb534ec6afd03..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04460904002189636,15.586040496826172,0.19916076958179474,800.0 -100,200,700,800,0.05220770835876465,13.942571640014648,0.24103039503097534,800.0 -200,300,800,900,0.058822568506002426,13.798318862915039,0.25405892729759216,800.0 -300,400,900,1000,0.05721569433808327,13.69599723815918,0.2527789771556854,800.0 -400,500,1000,1100,0.06382329761981964,13.144994735717773,0.30127647519111633,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 221e0a91e873b2b4f8a03a7357ebefe96f7fe0d2..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.049299053847789764,14.914196014404297,0.21670620143413544,800.0 -100,200,700,800,0.05569545924663544,13.637596130371094,0.254485160112381,800.0 -200,300,800,900,0.06601418554782867,13.175434112548828,0.2816983163356781,800.0 -300,400,900,1000,0.06528297066688538,13.307114601135254,0.27359893918037415,800.0 -400,500,1000,1100,0.0722796842455864,12.872069358825684,0.3159264624118805,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index d51986484b9b3c8f2f59ce586fe628598e42cf87..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04254748299717903,15.355696678161621,0.20510919392108917,800.0 -100,200,700,800,0.05110134184360504,14.24047565460205,0.23789489269256592,800.0 -200,300,800,900,0.06175930052995682,13.749947547912598,0.2514672577381134,800.0 -300,400,900,1000,0.06014503538608551,13.479216575622559,0.26726922392845154,800.0 -400,500,1000,1100,0.06997647881507874,12.888340950012207,0.32967787981033325,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index b7bafcb43ce489c082abb85980b5cc3de1456abd..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05189532786607742,14.840362548828125,0.20876067876815796,800.0 -100,200,700,800,0.060623060911893845,13.236340522766113,0.25940176844596863,800.0 -200,300,800,900,0.06716877222061157,13.0153226852417,0.2708689570426941,800.0 -300,400,900,1000,0.06950125098228455,12.456579208374023,0.2841930687427521,800.0 -400,500,1000,1100,0.08317962288856506,11.546085357666016,0.32316115498542786,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index ab3284497fee24ba4b41c5edf87324475cc43e25..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04325985908508301,15.80246353149414,0.19416770339012146,800.0 -100,200,700,800,0.05115324631333351,14.361542701721191,0.2324192374944687,800.0 -200,300,800,900,0.06286799162626266,13.555582046508789,0.2604959011077881,800.0 -300,400,900,1000,0.0565660186111927,13.948417663574219,0.24862140417099,800.0 -400,500,1000,1100,0.07273329049348831,12.927846908569336,0.31139159202575684,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index baac64aa68dc21f1bbf6bb678f1948d938b68520..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.042605914175510406,15.67961311340332,0.19789791107177734,800.0 -100,200,700,800,0.05370700731873512,13.917834281921387,0.24308547377586365,800.0 -200,300,800,900,0.06026593595743179,13.496360778808594,0.2607309818267822,800.0 -300,400,900,1000,0.058841291815042496,13.462464332580566,0.2699192464351654,800.0 -400,500,1000,1100,0.0724806934595108,12.392935752868652,0.32158568501472473,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index ad6db81d4bfedb7b7d9f7858b68a6c6d8cf22d7f..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03596346452832222,16.103160858154297,0.1861935257911682,800.0 -100,200,700,800,0.06807054579257965,13.654976844787598,0.2808187007904053,800.0 -200,300,800,900,0.07312279939651489,13.337336540222168,0.28557029366493225,800.0 -300,400,900,1000,0.06790318340063095,13.421137809753418,0.28165146708488464,800.0 -400,500,1000,1100,0.0722406655550003,12.980904579162598,0.31457412242889404,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index 3b39b424f5cd384ded38c05363dcca06d6776f67..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07004103064537048,13.74842643737793,0.29791462421417236,800.0 -100,200,700,800,0.10526213049888611,11.431946754455566,0.39493653178215027,800.0 -200,300,800,900,0.09230542927980423,12.435918807983398,0.3420048654079437,800.0 -300,400,900,1000,0.09044374525547028,11.93890380859375,0.3618695139884949,800.0 -400,500,1000,1100,0.1024089977145195,11.827131271362305,0.4036780595779419,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 98d8fcd79daf2731c7e55ec4fea5aa4d33cb55e3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04083194583654404,19.31778907775879,0.24085305631160736,800.0 -100,200,700,800,0.06421543657779694,16.17597198486328,0.35054177045822144,800.0 -200,300,800,900,0.09715315699577332,16.416183471679688,0.35015541315078735,800.0 -300,400,900,1000,0.1272880733013153,11.847212791442871,0.44196686148643494,800.0 -400,500,1000,1100,0.13948877155780792,13.807989120483398,0.4208296537399292,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index 498d759bffc831306287cf5aad30807ef4608818..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.028972728177905083,20.527101516723633,0.16853877902030945,800.0 -100,200,700,800,0.040970850735902786,18.064271926879883,0.2278379648923874,800.0 -200,300,800,900,0.0435137078166008,18.78623390197754,0.23804278671741486,800.0 -300,400,900,1000,0.06765207648277283,14.341693878173828,0.3159632086753845,800.0 -400,500,1000,1100,0.04365721717476845,17.434368133544922,0.2708764374256134,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index 20171b902ce734aa0deb60263fb733b377949a40..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024065766483545303,21.220888137817383,0.1476336270570755,800.0 -100,200,700,800,0.038543447852134705,18.531421661376953,0.2214019000530243,800.0 -200,300,800,900,0.038810327649116516,19.561447143554688,0.23122666776180267,800.0 -300,400,900,1000,0.05355850234627724,15.323894500732422,0.2685241401195526,800.0 -400,500,1000,1100,0.036472368985414505,18.775894165039062,0.23656055331230164,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index 1f982333b885e32bdf9c1393e80e79cd542e08c0..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026681050658226013,20.920238494873047,0.17142322659492493,800.0 -100,200,700,800,0.0569702610373497,17.48784637451172,0.2640688419342041,800.0 -200,300,800,900,0.07190109044313431,16.28997802734375,0.32379013299942017,800.0 -300,400,900,1000,0.07861140370368958,13.5723237991333,0.35313305258750916,800.0 -400,500,1000,1100,0.05471058189868927,15.874009132385254,0.31750601530075073,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 7bcfbe80a28d34f77376d773bab3d5ba0c1a4f2d..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026550330221652985,20.86431884765625,0.15692278742790222,800.0 -100,200,700,800,0.03950278088450432,18.778804779052734,0.2059803456068039,800.0 -200,300,800,900,0.037609923630952835,19.915128707885742,0.22249789535999298,800.0 -300,400,900,1000,0.05818944051861763,15.31616497039795,0.2743985950946808,800.0 -400,500,1000,1100,0.03549526631832123,18.919546127319336,0.21539458632469177,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index 0b3f480a2896d85a6dc2a9c3322fed307d46882e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02460501901805401,21.082042694091797,0.1401776522397995,800.0 -100,200,700,800,0.038622163236141205,18.92162322998047,0.19015733897686005,800.0 -200,300,800,900,0.0397634282708168,19.861082077026367,0.21123972535133362,800.0 -300,400,900,1000,0.0494094043970108,16.403799057006836,0.23035535216331482,800.0 -400,500,1000,1100,0.04680255800485611,17.80933380126953,0.23102176189422607,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index a5ad20d71c12046dab1d598948e70e8308ca047b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025219902396202087,20.667654037475586,0.1644982099533081,800.0 -100,200,700,800,0.030973227694630623,18.747764587402344,0.18947894871234894,800.0 -200,300,800,900,0.03562059625983238,19.665287017822266,0.21311891078948975,800.0 -300,400,900,1000,0.0448436439037323,15.638921737670898,0.2294638454914093,800.0 -400,500,1000,1100,0.03785225376486778,17.918909072875977,0.24320998787879944,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index a99f391a6f8b067c974e830bbfc90a4088bade3a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.033460162580013275,19.73522186279297,0.19191917777061462,800.0 -100,200,700,800,0.04936632513999939,17.611007690429688,0.2361345887184143,800.0 -200,300,800,900,0.06207498908042908,17.173612594604492,0.2737567126750946,800.0 -300,400,900,1000,0.060075730085372925,14.345711708068848,0.2940024137496948,800.0 -400,500,1000,1100,0.05672041326761246,16.957664489746094,0.2815020978450775,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index b7914380c3ff0ed744eee0190fa5d001bdec9450..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02207290381193161,21.287067413330078,0.1340504288673401,800.0 -100,200,700,800,0.036848921328783035,19.362930297851562,0.1794188916683197,800.0 -200,300,800,900,0.03279825299978256,20.037534713745117,0.19111938774585724,800.0 -300,400,900,1000,0.051445331424474716,15.905826568603516,0.21904927492141724,800.0 -400,500,1000,1100,0.03951036557555199,18.930326461791992,0.21755450963974,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 2e3533d697cd0a810b79f781b9a440431333b65e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02489827387034893,20.880264282226562,0.14155836403369904,800.0 -100,200,700,800,0.044648267328739166,18.40045928955078,0.20612022280693054,800.0 -200,300,800,900,0.05398062616586685,18.540794372558594,0.2444291114807129,800.0 -300,400,900,1000,0.054239362478256226,15.04028606414795,0.2619078755378723,800.0 -400,500,1000,1100,0.04107142612338066,17.904176712036133,0.23754799365997314,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 40d521cc6357091366a65b3c782b2c1afdbce9ab..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03392269089818001,20.645221710205078,0.1463446021080017,800.0 -100,200,700,800,0.05444112792611122,18.034273147583008,0.21471750736236572,800.0 -200,300,800,900,0.06142718717455864,18.12811851501465,0.24073825776576996,800.0 -300,400,900,1000,0.060297705233097076,15.120783805847168,0.22731228172779083,800.0 -400,500,1000,1100,0.07067456841468811,15.27718734741211,0.2648458480834961,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 1cab7cb920accdfa58aa5b2ddf3a332d60a8ea9e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03172869607806206,20.25861167907715,0.1764363795518875,800.0 -100,200,700,800,0.05356694757938385,18.113758087158203,0.21829824149608612,800.0 -200,300,800,900,0.04178520664572716,19.82250213623047,0.2153877317905426,800.0 -300,400,900,1000,0.04955707862973213,16.21304702758789,0.20210790634155273,800.0 -400,500,1000,1100,0.06886377185583115,16.169374465942383,0.2568584680557251,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index 2c9786aa5de3a1a9c45cea9f60f3a458a785c200..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023743517696857452,21.308032989501953,0.13663049042224884,800.0 -100,200,700,800,0.035580649971961975,19.043132781982422,0.1798095703125,800.0 -200,300,800,900,0.04021226242184639,19.447952270507812,0.2048248052597046,800.0 -300,400,900,1000,0.04154418036341667,16.13603973388672,0.19677919149398804,800.0 -400,500,1000,1100,0.03914010897278786,17.205211639404297,0.23552833497524261,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index 0f8d7e8e1b1dbe5fb64a854a92a20b86c5b42d64..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023008059710264206,21.257490158081055,0.14081315696239471,800.0 -100,200,700,800,0.03674706444144249,18.471485137939453,0.19211579859256744,800.0 -200,300,800,900,0.04047909751534462,19.202926635742188,0.20663602650165558,800.0 -300,400,900,1000,0.05069038271903992,15.998726844787598,0.24198301136493683,800.0 -400,500,1000,1100,0.05740383267402649,17.284629821777344,0.24625913798809052,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 0328b632034f0ced095d286973b79566061b1761..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022346515208482742,21.344284057617188,0.13827446103096008,800.0 -100,200,700,800,0.03949974849820137,18.4444522857666,0.19284622371196747,800.0 -200,300,800,900,0.03553430363535881,19.98480796813965,0.20414453744888306,800.0 -300,400,900,1000,0.04790092632174492,15.640713691711426,0.2335124909877777,800.0 -400,500,1000,1100,0.03333917260169983,19.10987091064453,0.21956509351730347,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index 169e6f970f1f68e6830e6a9d5b820e84d3247de8..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02478119358420372,21.2519474029541,0.13563406467437744,800.0 -100,200,700,800,0.027972321957349777,19.6452579498291,0.1719367355108261,800.0 -200,300,800,900,0.029110345989465714,20.350379943847656,0.1881389170885086,800.0 -300,400,900,1000,0.03334492817521095,16.99921989440918,0.17760325968265533,800.0 -400,500,1000,1100,0.028242330998182297,19.079227447509766,0.19473996758460999,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index e4fdae062e06c7f99f1c8581b6ab38a452652a9d..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024584662169218063,20.832082748413086,0.14750468730926514,800.0 -100,200,700,800,0.048170577734708786,18.230567932128906,0.19625410437583923,800.0 -200,300,800,900,0.039468102157115936,18.91514778137207,0.2093711793422699,800.0 -300,400,900,1000,0.05109180137515068,15.7702054977417,0.2274131029844284,800.0 -400,500,1000,1100,0.03950752690434456,17.829214096069336,0.23599691689014435,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index 2a2d358f6e4ea3ae82e86b4812b05309b6d4aefb..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024784503504633904,21.099933624267578,0.13939034938812256,800.0 -100,200,700,800,0.03281332924962044,19.082130432128906,0.19849704205989838,800.0 -200,300,800,900,0.06342510879039764,17.25360870361328,0.25430116057395935,800.0 -300,400,900,1000,0.0405069924890995,15.994916915893555,0.21651215851306915,800.0 -400,500,1000,1100,0.03418212756514549,18.742603302001953,0.22114010155200958,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index c8d5bea4d2c47c61263b4377e2f0aab7bff44258..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02906530164182186,20.962923049926758,0.16013456881046295,800.0 -100,200,700,800,0.036155782639980316,18.83992576599121,0.18676109611988068,800.0 -200,300,800,900,0.033688053488731384,20.159711837768555,0.19242477416992188,800.0 -300,400,900,1000,0.04552407190203667,15.63217544555664,0.25653353333473206,800.0 -400,500,1000,1100,0.04132045432925224,17.660367965698242,0.2290736585855484,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index a1f7d60fa3b929f40f727fc23042cdd561c618a4..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022246647626161575,21.107284545898438,0.14345794916152954,800.0 -100,200,700,800,0.04396833851933479,18.400619506835938,0.20474547147750854,800.0 -200,300,800,900,0.039137549698352814,19.308002471923828,0.21602247655391693,800.0 -300,400,900,1000,0.044323455542325974,16.34917449951172,0.21315595507621765,800.0 -400,500,1000,1100,0.03456170856952667,19.034303665161133,0.23493152856826782,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index f53c74d42ba4d62bf3e512a8c818b7915ecd0e0b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022118903696537018,21.4188289642334,0.14778254926204681,800.0 -100,200,700,800,0.03643392398953438,19.634376525878906,0.17926625907421112,800.0 -200,300,800,900,0.04395642131567001,18.842967987060547,0.21089869737625122,800.0 -300,400,900,1000,0.047699201852083206,16.170385360717773,0.20774617791175842,800.0 -400,500,1000,1100,0.026233576238155365,19.467754364013672,0.1919575184583664,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index e78feba8b21117554aaff900e65773b13eb09ff1..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02140287309885025,21.463769912719727,0.1339161992073059,800.0 -100,200,700,800,0.030130065977573395,19.598852157592773,0.17601926624774933,800.0 -200,300,800,900,0.03085358440876007,20.05951499938965,0.1878638118505478,800.0 -300,400,900,1000,0.03518277034163475,17.082229614257812,0.18369056284427643,800.0 -400,500,1000,1100,0.033458780497312546,19.010190963745117,0.2086406946182251,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index 348aac9d3dd919669c8f6476c41559af37b32880..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021668802946805954,21.635330200195312,0.13421578705310822,800.0 -100,200,700,800,0.039461053907871246,18.391380310058594,0.209271639585495,800.0 -200,300,800,900,0.0337190181016922,19.882627487182617,0.21659477055072784,800.0 -300,400,900,1000,0.049786731600761414,15.227583885192871,0.27440890669822693,800.0 -400,500,1000,1100,0.03422364965081215,18.080291748046875,0.25728797912597656,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index c0e37b2f640da722b5f29a92849fa0e99c1ab26b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025973796844482422,20.939268112182617,0.14481836557388306,800.0 -100,200,700,800,0.029675068333745003,19.324745178222656,0.1727224737405777,800.0 -200,300,800,900,0.03003893606364727,20.67168617248535,0.18578359484672546,800.0 -300,400,900,1000,0.0335078090429306,17.34571647644043,0.17335638403892517,800.0 -400,500,1000,1100,0.03272290900349617,18.588960647583008,0.1991923749446869,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 274d8e703ec4adb0deeec03ee2f64a5a63ad0164..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02395821548998356,21.60665512084961,0.13049747049808502,800.0 -100,200,700,800,0.03965659439563751,19.337932586669922,0.1759856790304184,800.0 -200,300,800,900,0.029306482523679733,20.619140625,0.17950916290283203,800.0 -300,400,900,1000,0.04102053493261337,16.5142765045166,0.1894548237323761,800.0 -400,500,1000,1100,0.03500564396381378,18.563640594482422,0.20889516174793243,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 3f5d6bd38ab397dbc2af89b749a0fd7a8b4423b5..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02188713289797306,21.55044937133789,0.12526319921016693,800.0 -100,200,700,800,0.04697217047214508,18.062477111816406,0.20103149116039276,800.0 -200,300,800,900,0.046005092561244965,18.947385787963867,0.2319970279932022,800.0 -300,400,900,1000,0.04150402918457985,16.789386749267578,0.1962798833847046,800.0 -400,500,1000,1100,0.0404619462788105,17.67552947998047,0.23680847883224487,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index 929f263d5ee758193497f85bc53085cef629f33c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020433394238352776,21.772581100463867,0.12484397739171982,800.0 -100,200,700,800,0.040874823927879333,19.129453659057617,0.1900891661643982,800.0 -200,300,800,900,0.03248322382569313,20.04485511779785,0.19206708669662476,800.0 -300,400,900,1000,0.036983098834753036,16.67238426208496,0.18400827050209045,800.0 -400,500,1000,1100,0.031024591997265816,18.36117935180664,0.20656436681747437,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index d576cb3aa916ffc976852e93b0f3d72bb2e62855..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022816691547632217,21.653783798217773,0.13391344249248505,800.0 -100,200,700,800,0.039462219923734665,19.233949661254883,0.1841772496700287,800.0 -200,300,800,900,0.037337176501750946,20.12301254272461,0.2127307951450348,800.0 -300,400,900,1000,0.05039737746119499,15.963687896728516,0.23348639905452728,800.0 -400,500,1000,1100,0.034182485193014145,18.399084091186523,0.2253739982843399,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index da1b7aebff43df52d5e6a139a26438f9a1efc649..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02363365888595581,21.211353302001953,0.12994036078453064,800.0 -100,200,700,800,0.034190818667411804,18.877408981323242,0.18282148241996765,800.0 -200,300,800,900,0.03498193621635437,19.658960342407227,0.20237725973129272,800.0 -300,400,900,1000,0.03823167458176613,16.662155151367188,0.1858067661523819,800.0 -400,500,1000,1100,0.029699385166168213,19.50615882873535,0.19592130184173584,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index 0e07c36a58683179b322505445b1468811033542..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022637611255049706,21.40622329711914,0.12377378344535828,800.0 -100,200,700,800,0.034985195845365524,19.12494468688965,0.1874980479478836,800.0 -200,300,800,900,0.04796417057514191,19.736711502075195,0.2165532261133194,800.0 -300,400,900,1000,0.04655729606747627,16.907968521118164,0.2086734026670456,800.0 -400,500,1000,1100,0.047551002353429794,18.10275650024414,0.21738475561141968,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 9f2850c6777d6155e3118709467992c6fb6de53a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02103429101407528,21.42401123046875,0.12967254221439362,800.0 -100,200,700,800,0.027861280366778374,19.252151489257812,0.16832394897937775,800.0 -200,300,800,900,0.03613884001970291,19.425743103027344,0.19276143610477448,800.0 -300,400,900,1000,0.033000264316797256,17.430810928344727,0.1705416887998581,800.0 -400,500,1000,1100,0.03869181498885155,18.092613220214844,0.20823106169700623,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index 7504420931d045cae03d9367ebcf4207878065cb..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026584351435303688,20.161880493164062,0.1489136666059494,800.0 -100,200,700,800,0.04170206934213638,18.780981063842773,0.1876697540283203,800.0 -200,300,800,900,0.06290731579065323,16.535633087158203,0.24078932404518127,800.0 -300,400,900,1000,0.05005541816353798,15.8792724609375,0.2073979377746582,800.0 -400,500,1000,1100,0.06279882788658142,16.31987953186035,0.2555239796638489,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index c2cdac3159427e1c723bb2947aed96fb1d50a2da..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026972029358148575,21.01654052734375,0.15185900032520294,800.0 -100,200,700,800,0.0394805371761322,18.785221099853516,0.18447932600975037,800.0 -200,300,800,900,0.039168547838926315,19.426345825195312,0.20873598754405975,800.0 -300,400,900,1000,0.04701012000441551,16.604459762573242,0.21223390102386475,800.0 -400,500,1000,1100,0.03691202774643898,18.461088180541992,0.2269073873758316,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index e27c7a1b0876179d057f4dd31e2c16231971f22b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019387930631637573,21.647985458374023,0.13860207796096802,800.0 -100,200,700,800,0.029881076887249947,19.8074893951416,0.18108795583248138,800.0 -200,300,800,900,0.033971622586250305,19.343305587768555,0.19767116010189056,800.0 -300,400,900,1000,0.036918893456459045,17.085830688476562,0.19518901407718658,800.0 -400,500,1000,1100,0.04037541523575783,18.61392593383789,0.22860348224639893,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index aa8c81cbbf5f36a9161dc34b3d0fcb05f37f61b1..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.030716121196746826,20.71499252319336,0.14724300801753998,800.0 -100,200,700,800,0.06400851160287857,16.874492645263672,0.22445568442344666,800.0 -200,300,800,900,0.044783901423215866,18.750219345092773,0.22560033202171326,800.0 -300,400,900,1000,0.05204049125313759,15.13215446472168,0.24192088842391968,800.0 -400,500,1000,1100,0.04752305895090103,16.469558715820312,0.258280873298645,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index a1db747c1c336d2e5643f637df5c324b968bd94a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0227956585586071,20.343381881713867,0.1436024159193039,800.0 -100,200,700,800,0.029878783971071243,19.63530731201172,0.18105000257492065,800.0 -200,300,800,900,0.033383358269929886,19.344501495361328,0.19175466895103455,800.0 -300,400,900,1000,0.03575250133872032,17.061050415039062,0.17696774005889893,800.0 -400,500,1000,1100,0.03468412160873413,18.9685115814209,0.1880340576171875,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 7cd73fba1b8a70e39d5ec14dcb80dda7448f4266..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025614051148295403,21.505342483520508,0.13216589391231537,800.0 -100,200,700,800,0.038207169622182846,19.579824447631836,0.18196485936641693,800.0 -200,300,800,900,0.03356423228979111,20.110095977783203,0.1857714056968689,800.0 -300,400,900,1000,0.05020948499441147,17.066020965576172,0.1946629285812378,800.0 -400,500,1000,1100,0.023705752566456795,20.022743225097656,0.18570873141288757,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 9e403273864d7efc5fcdda8426de19eac27b59b3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018134670332074165,22.00558853149414,0.1169581189751625,800.0 -100,200,700,800,0.026300640776753426,20.090362548828125,0.1475110650062561,800.0 -200,300,800,900,0.033022526651620865,19.932708740234375,0.18304546177387238,800.0 -300,400,900,1000,0.037887703627347946,16.7653751373291,0.18164663016796112,800.0 -400,500,1000,1100,0.044011540710926056,17.62427520751953,0.2159174531698227,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 237b55de944ba50cb1e1dbc4e81bddd3df03cdb3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260706_215204-qv85iql0/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027587080374360085,21.151512145996094,0.1373169720172882,800.0 -100,200,700,800,0.05843239650130272,18.287887573242188,0.2064676284790039,800.0 -200,300,800,900,0.040584832429885864,18.484912872314453,0.1953093558549881,800.0 -300,400,900,1000,0.04552861303091049,16.51851463317871,0.18664108216762543,800.0 -400,500,1000,1100,0.042944520711898804,18.37255096435547,0.21644267439842224,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 74d7ed18ca3add9bd1c5374bd8d0df6a75eab183..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02229025401175022,21.319263458251953,0.13519896566867828,800.0 -100,200,700,800,0.05519429221749306,17.714826583862305,0.20771300792694092,800.0 -200,300,800,900,0.06739741563796997,17.947242736816406,0.23289676010608673,800.0 -300,400,900,1000,0.05251266807317734,16.3505916595459,0.19887486100196838,800.0 -400,500,1000,1100,0.0654379278421402,17.19559669494629,0.23194947838783264,800.0 diff --git a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index 81212807df0cad7c04a6b6ebdda22c4d5aea6026..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025213584303855896,20.790878295898438,0.1627865582704544,800.0 -100,200,700,800,0.032923147082328796,19.180152893066406,0.20110920071601868,800.0 -200,300,800,900,0.03293067216873169,19.5159969329834,0.20828010141849518,800.0 -300,400,900,1000,0.035161420702934265,16.73447608947754,0.18305785953998566,800.0 -400,500,1000,1100,0.028123950585722923,19.135120391845703,0.20085710287094116,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00002000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00002000.csv deleted file mode 100644 index dfbfa2ade2c5e5057a308a6ca3f0fa914e1b36d0..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00002000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.07541943341493607,12.431258201599121,0.3340534567832947,800.0 -100,200,700,800,0.09553022682666779,10.811829566955566,0.4131854176521301,800.0 -200,300,800,900,0.0969967395067215,10.929692268371582,0.4098820388317108,800.0 -300,400,900,1000,0.08858799189329147,11.312546730041504,0.3963899314403534,800.0 -400,500,1000,1100,0.09727829694747925,10.621933937072754,0.4555432200431824,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00004000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00004000.csv deleted file mode 100644 index a812d93bb0303ed7aa03da0c2f9003b6f7382581..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00004000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06851114332675934,13.016297340393066,0.30045783519744873,800.0 -100,200,700,800,0.07977868616580963,11.599736213684082,0.36737000942230225,800.0 -200,300,800,900,0.07994730770587921,11.921943664550781,0.37042129039764404,800.0 -300,400,900,1000,0.08446288108825684,11.480310440063477,0.3745356798171997,800.0 -400,500,1000,1100,0.08776773512363434,11.199995994567871,0.41472625732421875,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00006000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00006000.csv deleted file mode 100644 index a32399894f0b9264d7d203e358800820d7652d70..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00006000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05593676492571831,13.665265083312988,0.2729450464248657,800.0 -100,200,700,800,0.08696575462818146,11.477948188781738,0.3562753200531006,800.0 -200,300,800,900,0.08494265377521515,11.631616592407227,0.3457280099391937,800.0 -300,400,900,1000,0.09184189885854721,11.281389236450195,0.35848402976989746,800.0 -400,500,1000,1100,0.08793214708566666,11.422920227050781,0.3874075412750244,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00008000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00008000.csv deleted file mode 100644 index 51a9794b6fe9bb4c2030073a6a9ea054963b409e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00008000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.05668633058667183,13.785715103149414,0.2681330740451813,800.0 -100,200,700,800,0.08260596543550491,11.705702781677246,0.3468746840953827,800.0 -200,300,800,900,0.06996304541826248,12.308396339416504,0.30642905831336975,800.0 -300,400,900,1000,0.08387599885463715,11.865167617797852,0.35615867376327515,800.0 -400,500,1000,1100,0.0798923596739769,12.048855781555176,0.3785910904407501,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00010000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00010000.csv deleted file mode 100644 index b54939b9d456a631a25509a1876d78c3343f906e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00010000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.050745561718940735,14.41677474975586,0.23878999054431915,800.0 -100,200,700,800,0.06295131891965866,12.847673416137695,0.28623297810554504,800.0 -200,300,800,900,0.07045076787471771,12.46875,0.31282883882522583,800.0 -300,400,900,1000,0.07261668890714645,12.48668098449707,0.3172306716442108,800.0 -400,500,1000,1100,0.08058197051286697,12.020628929138184,0.35657423734664917,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00012000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00012000.csv deleted file mode 100644 index 62f9900ba02897135c4cacdcc953e9404bffe782..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00012000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.052764467895030975,14.113844871520996,0.23700910806655884,800.0 -100,200,700,800,0.06158691272139549,12.865221977233887,0.28272584080696106,800.0 -200,300,800,900,0.06622277200222015,12.526327133178711,0.29515257477760315,800.0 -300,400,900,1000,0.06026114895939827,13.083473205566406,0.2823679447174072,800.0 -400,500,1000,1100,0.0793423056602478,11.653027534484863,0.35161322355270386,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00014000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00014000.csv deleted file mode 100644 index 82f91a729403c1151109cd9dd7a1ca787d3746f6..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00014000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.046028438955545425,14.89044189453125,0.21588870882987976,800.0 -100,200,700,800,0.06219041347503662,13.339999198913574,0.25646349787712097,800.0 -200,300,800,900,0.07071784138679504,12.986113548278809,0.26608285307884216,800.0 -300,400,900,1000,0.05742011219263077,13.699633598327637,0.24494193494319916,800.0 -400,500,1000,1100,0.06583471596240997,13.068408966064453,0.2965083420276642,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00016000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00016000.csv deleted file mode 100644 index 83eea666be283062e9cc9feeef764a8a221c1972..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00016000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04423563927412033,15.452588081359863,0.20214757323265076,800.0 -100,200,700,800,0.05415288358926773,13.837410926818848,0.23898041248321533,800.0 -200,300,800,900,0.06085387244820595,13.44388198852539,0.2662532925605774,800.0 -300,400,900,1000,0.07017764449119568,13.135645866394043,0.28427207469940186,800.0 -400,500,1000,1100,0.0684625655412674,12.9295654296875,0.3102996349334717,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00018000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00018000.csv deleted file mode 100644 index f1340054a1cd1b452f5c1cc0182edfcd17b22861..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00018000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.040271490812301636,15.660558700561523,0.19039508700370789,800.0 -100,200,700,800,0.048124074935913086,14.360834121704102,0.22636283934116364,800.0 -200,300,800,900,0.05404625087976456,13.832858085632324,0.2490895390510559,800.0 -300,400,900,1000,0.05519842728972435,13.661787033081055,0.26671355962753296,800.0 -400,500,1000,1100,0.06971941143274307,12.795571327209473,0.3226661682128906,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00020000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00020000.csv deleted file mode 100644 index 888f89381bd3a521d5bd0ba1a64f54b6afeb1298..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00020000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04657641053199768,15.279186248779297,0.20398931205272675,800.0 -100,200,700,800,0.05861348286271095,13.493741035461426,0.25236085057258606,800.0 -200,300,800,900,0.060901954770088196,13.357975006103516,0.27203550934791565,800.0 -300,400,900,1000,0.06573709100484848,12.923810958862305,0.285999596118927,800.0 -400,500,1000,1100,0.07047252357006073,12.763876914978027,0.31484970450401306,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00022000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00022000.csv deleted file mode 100644 index 776851c13d2dcaa32790b58a3b7ebff0bf76aa35..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00022000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.042767733335494995,15.4757661819458,0.19890853762626648,800.0 -100,200,700,800,0.05790546536445618,13.464516639709473,0.2630457282066345,800.0 -200,300,800,900,0.05933889374136925,13.277624130249023,0.2710109055042267,800.0 -300,400,900,1000,0.06608053296804428,12.761736869812012,0.2781533896923065,800.0 -400,500,1000,1100,0.07998285442590714,11.985091209411621,0.33675575256347656,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00024000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00024000.csv deleted file mode 100644 index 2b1b25cd160ea3d9abde70ad10df17e588372f23..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00024000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04563787579536438,15.32179069519043,0.2026631385087967,800.0 -100,200,700,800,0.053773440420627594,13.887495994567871,0.2416178584098816,800.0 -200,300,800,900,0.06378351897001266,13.377740859985352,0.26847633719444275,800.0 -300,400,900,1000,0.07121498882770538,12.691662788391113,0.3027380406856537,800.0 -400,500,1000,1100,0.08407701551914215,12.352944374084473,0.3405129909515381,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00026000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00026000.csv deleted file mode 100644 index 65ea45875172fe3d451a788a0dd98d30330142a4..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00026000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.04051213338971138,15.705785751342773,0.19295309484004974,800.0 -100,200,700,800,0.05027681216597557,14.005091667175293,0.23834580183029175,800.0 -200,300,800,900,0.05602053552865982,13.910660743713379,0.2571139931678772,800.0 -300,400,900,1000,0.06454700231552124,13.38757610321045,0.2804979979991913,800.0 -400,500,1000,1100,0.0634072795510292,13.100970268249512,0.3130303621292114,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00028000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00028000.csv deleted file mode 100644 index 69bb54b037ebd9f83120436e0029fd99296a7cb4..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00028000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03751993924379349,16.048276901245117,0.19083362817764282,800.0 -100,200,700,800,0.052511390298604965,14.169533729553223,0.24465049803256989,800.0 -200,300,800,900,0.06268373131752014,13.348554611206055,0.2628314197063446,800.0 -300,400,900,1000,0.05441734939813614,13.528255462646484,0.25010430812835693,800.0 -400,500,1000,1100,0.07222132384777069,12.705493927001953,0.3142305016517639,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00030000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00030000.csv deleted file mode 100644 index 2a8b8828f09977e182c6bdd166e46614dabd9356..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00030000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.041395097970962524,15.598977088928223,0.2024458348751068,800.0 -100,200,700,800,0.05469619110226631,13.700671195983887,0.24980002641677856,800.0 -200,300,800,900,0.05830841138958931,13.80743408203125,0.25457683205604553,800.0 -300,400,900,1000,0.05570003390312195,13.709918975830078,0.2654293179512024,800.0 -400,500,1000,1100,0.06930077821016312,12.919347763061523,0.3058885335922241,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00032000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00032000.csv deleted file mode 100644 index dee780c5f372414f03be82ba7a409d9f75fc2446..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00032000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03497093543410301,16.23996353149414,0.1769687831401825,800.0 -100,200,700,800,0.0414925180375576,15.25891399383545,0.20522773265838623,800.0 -200,300,800,900,0.05322747305035591,14.357280731201172,0.23380257189273834,800.0 -300,400,900,1000,0.05179388076066971,14.11041259765625,0.2515086233615875,800.0 -400,500,1000,1100,0.08352547883987427,12.336109161376953,0.3669067621231079,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00034000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00034000.csv deleted file mode 100644 index 8b7adacf4bed954d17921b219991c0bd1ebf4024..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00034000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.040270619094371796,16.181325912475586,0.18980783224105835,800.0 -100,200,700,800,0.06026682257652283,13.945456504821777,0.24797554314136505,800.0 -200,300,800,900,0.08137484639883041,13.270277976989746,0.29363587498664856,800.0 -300,400,900,1000,0.10675857216119766,12.342859268188477,0.3151673972606659,800.0 -400,500,1000,1100,0.12049224972724915,11.793150901794434,0.3374299705028534,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00036000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00036000.csv deleted file mode 100644 index 24d8162bccc8b16eda185cc6abff4088592ef296..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00036000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03720177710056305,16.40703010559082,0.17532867193222046,800.0 -100,200,700,800,0.04552222415804863,14.77905559539795,0.21385833621025085,800.0 -200,300,800,900,0.050898417830467224,14.466238975524902,0.227792888879776,800.0 -300,400,900,1000,0.051092762500047684,14.516966819763184,0.23221783339977264,800.0 -400,500,1000,1100,0.058513153344392776,13.844735145568848,0.2749321460723877,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00038000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00038000.csv deleted file mode 100644 index 33e4125f432d63460f08680fc43f14a05a8ab397..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00038000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03542128950357437,16.435508728027344,0.1778961420059204,800.0 -100,200,700,800,0.039484903216362,15.354387283325195,0.20644664764404297,800.0 -200,300,800,900,0.050936803221702576,14.503456115722656,0.23403862118721008,800.0 -300,400,900,1000,0.04678993672132492,14.555368423461914,0.22528629004955292,800.0 -400,500,1000,1100,0.06594674289226532,13.221845626831055,0.28277289867401123,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00040000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00040000.csv deleted file mode 100644 index f9d72a288e721d32bfaaa7f496d7de3556ed0945..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00040000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.06748280674219131,14.254866600036621,0.28640854358673096,800.0 -100,200,700,800,0.08474013209342957,13.055673599243164,0.3660092055797577,800.0 -200,300,800,900,0.07300100475549698,12.718379974365234,0.34329986572265625,800.0 -300,400,900,1000,0.0946110412478447,11.75575065612793,0.3878406286239624,800.0 -400,500,1000,1100,0.0759463980793953,12.942795753479004,0.37184983491897583,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00042000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00042000.csv deleted file mode 100644 index 21a76a38c943f5cada0a3b9653dc5748faa23b33..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00042000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.031511981040239334,20.151203155517578,0.17184020578861237,800.0 -100,200,700,800,0.05687132850289345,15.701685905456543,0.2840375602245331,800.0 -200,300,800,900,0.04533714801073074,18.074018478393555,0.2643892765045166,800.0 -300,400,900,1000,0.06780794262886047,13.263052940368652,0.32325470447540283,800.0 -400,500,1000,1100,0.07026427984237671,14.822872161865234,0.33096131682395935,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00044000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00044000.csv deleted file mode 100644 index d961115997f41bad4c49f071fcddd2b8e8cff363..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00044000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.041342392563819885,19.152803421020508,0.2073473334312439,800.0 -100,200,700,800,0.05304815247654915,16.960285186767578,0.22902674973011017,800.0 -200,300,800,900,0.04636317864060402,18.899127960205078,0.23488759994506836,800.0 -300,400,900,1000,0.062021151185035706,14.422710418701172,0.28940778970718384,800.0 -400,500,1000,1100,0.05218026787042618,16.588525772094727,0.2685493528842926,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00046000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00046000.csv deleted file mode 100644 index dec2620bcce9abdd7725e740e50f10f0d0b53e99..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00046000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.036177679896354675,20.027915954589844,0.18984243273735046,800.0 -100,200,700,800,0.05955687165260315,16.885313034057617,0.2482193410396576,800.0 -200,300,800,900,0.03736615553498268,19.704988479614258,0.23575882613658905,800.0 -300,400,900,1000,0.04571502283215523,15.560111045837402,0.24405193328857422,800.0 -400,500,1000,1100,0.037929728627204895,18.635889053344727,0.22782596945762634,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00048000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00048000.csv deleted file mode 100644 index 7b69c028e6b42df8be2094800fa002296c42b213..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00048000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029704151675105095,20.873640060424805,0.15852852165699005,800.0 -100,200,700,800,0.07872772961854935,16.372577667236328,0.2639197111129761,800.0 -200,300,800,900,0.05653145909309387,18.029834747314453,0.28536534309387207,800.0 -300,400,900,1000,0.06958068907260895,13.924291610717773,0.33198943734169006,800.0 -400,500,1000,1100,0.051686692982912064,16.115053176879883,0.3073706328868866,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00050000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00050000.csv deleted file mode 100644 index 43d08a43208871dd3ca1ad867d968e7c9f265716..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00050000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03952585533261299,20.017107009887695,0.18616212904453278,800.0 -100,200,700,800,0.05678999796509743,17.42821502685547,0.2378612905740738,800.0 -200,300,800,900,0.05586744472384453,17.579275131225586,0.2492315173149109,800.0 -300,400,900,1000,0.05256742611527443,15.185747146606445,0.251810759305954,800.0 -400,500,1000,1100,0.058396175503730774,16.5316219329834,0.2610899806022644,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00052000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00052000.csv deleted file mode 100644 index e7d9abdd3d73207ca51a1ea8ae99d0b6ec90f109..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00052000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.029937999323010445,20.2052059173584,0.16971541941165924,800.0 -100,200,700,800,0.03360763192176819,18.880409240722656,0.18913711607456207,800.0 -200,300,800,900,0.033540572971105576,19.890140533447266,0.20082508027553558,800.0 -300,400,900,1000,0.04399268329143524,16.2700138092041,0.2155143767595291,800.0 -400,500,1000,1100,0.027297141030430794,19.182031631469727,0.21230801939964294,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00054000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00054000.csv deleted file mode 100644 index 37c90756e226fe5273f666112fa2c2cbbcaede19..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00054000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02611621469259262,20.930334091186523,0.1573575884103775,800.0 -100,200,700,800,0.03755197674036026,18.997264862060547,0.19673489034175873,800.0 -200,300,800,900,0.034659840166568756,20.153200149536133,0.19655907154083252,800.0 -300,400,900,1000,0.03688214719295502,16.722999572753906,0.19346484541893005,800.0 -400,500,1000,1100,0.03492184355854988,17.889236450195312,0.21306844055652618,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00056000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00056000.csv deleted file mode 100644 index 8f032774f5ffce145964d3780267233f9380decd..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00056000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02997618168592453,20.4318904876709,0.15519815683364868,800.0 -100,200,700,800,0.03803873434662819,18.213275909423828,0.20928838849067688,800.0 -200,300,800,900,0.04398353397846222,19.244892120361328,0.23968319594860077,800.0 -300,400,900,1000,0.04812735691666603,15.806899070739746,0.22491943836212158,800.0 -400,500,1000,1100,0.030041219666600227,19.011985778808594,0.2024429738521576,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00058000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00058000.csv deleted file mode 100644 index aaea510b4242a37f83a129e880ed41794652d154..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00058000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020835908129811287,21.63603973388672,0.12391183525323868,800.0 -100,200,700,800,0.03661506250500679,18.57906723022461,0.19457203149795532,800.0 -200,300,800,900,0.043527666479349136,19.02149200439453,0.22236794233322144,800.0 -300,400,900,1000,0.05480566620826721,15.47973918914795,0.24919749796390533,800.0 -400,500,1000,1100,0.05508335679769516,16.490293502807617,0.27381542325019836,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00060000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00060000.csv deleted file mode 100644 index 9cee3bbe018ee4eb0cda8c913800123820d928da..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00060000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.03204857185482979,20.207008361816406,0.15896178781986237,800.0 -100,200,700,800,0.052161332219839096,17.39940071105957,0.21884578466415405,800.0 -200,300,800,900,0.04055088013410568,19.55597686767578,0.22946514189243317,800.0 -300,400,900,1000,0.07446523010730743,13.764413833618164,0.33889442682266235,800.0 -400,500,1000,1100,0.03439228981733322,18.207487106323242,0.24558930099010468,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00062000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00062000.csv deleted file mode 100644 index 66de1ba0904c56b73224d328f76472c0bc8cec0a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00062000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027087703347206116,20.394193649291992,0.1646653711795807,800.0 -100,200,700,800,0.04630771279335022,18.05540657043457,0.2263677567243576,800.0 -200,300,800,900,0.044362183660268784,18.734899520874023,0.2368786484003067,800.0 -300,400,900,1000,0.057991065084934235,15.218942642211914,0.2703736424446106,800.0 -400,500,1000,1100,0.05572642385959625,16.44866180419922,0.2880392372608185,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00064000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00064000.csv deleted file mode 100644 index 95ddb4a3eba0df2004ae48158f9aa0db66ceed9b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00064000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023710081353783607,21.09632110595703,0.1335974782705307,800.0 -100,200,700,800,0.030663160607218742,19.052011489868164,0.18845424056053162,800.0 -200,300,800,900,0.036244358867406845,19.857017517089844,0.22454631328582764,800.0 -300,400,900,1000,0.05335678160190582,15.855135917663574,0.24874523282051086,800.0 -400,500,1000,1100,0.03586915135383606,18.12288475036621,0.2607066333293915,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00066000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00066000.csv deleted file mode 100644 index fbf8e7de8a19d4dd1aa557e7ec7dc9809ec0266f..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00066000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021906595677137375,21.438486099243164,0.13633772730827332,800.0 -100,200,700,800,0.03677114099264145,19.005876541137695,0.195871964097023,800.0 -200,300,800,900,0.03931330144405365,19.60334587097168,0.228109210729599,800.0 -300,400,900,1000,0.051286593079566956,15.264004707336426,0.2574610114097595,800.0 -400,500,1000,1100,0.0387851819396019,17.95691680908203,0.26249682903289795,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00068000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00068000.csv deleted file mode 100644 index d9e087f397e8e4a37c02c67b32f36b2546130ed6..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00068000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026265911757946014,20.748580932617188,0.137067973613739,800.0 -100,200,700,800,0.03347691893577576,19.093107223510742,0.18182754516601562,800.0 -200,300,800,900,0.03773051127791405,19.388017654418945,0.2016335427761078,800.0 -300,400,900,1000,0.04575316607952118,15.830842018127441,0.21888501942157745,800.0 -400,500,1000,1100,0.026341145858168602,19.2489070892334,0.1992540955543518,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00070000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00070000.csv deleted file mode 100644 index 6cc413545fd6d12d3620f53b19616d47478b6cbf..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00070000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02782580442726612,20.933368682861328,0.14689140021800995,800.0 -100,200,700,800,0.036036159843206406,18.891132354736328,0.19690126180648804,800.0 -200,300,800,900,0.03644995391368866,19.914962768554688,0.20635433495044708,800.0 -300,400,900,1000,0.05527029186487198,15.574759483337402,0.2464575618505478,800.0 -400,500,1000,1100,0.04726823419332504,17.45688247680664,0.24240826070308685,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00072000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00072000.csv deleted file mode 100644 index 7cad257fe0a43ea1ab63517d381f76289ed488f5..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00072000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02204328589141369,21.295930862426758,0.13235202431678772,800.0 -100,200,700,800,0.027972083538770676,19.11435890197754,0.17559285461902618,800.0 -200,300,800,900,0.02707669697701931,19.88918113708496,0.1861288845539093,800.0 -300,400,900,1000,0.04370095580816269,16.042142868041992,0.20436854660511017,800.0 -400,500,1000,1100,0.025085076689720154,18.91480255126953,0.17793582379817963,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00074000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00074000.csv deleted file mode 100644 index 636da6a0de3a054946c8f9dc2331b046ca6a6206..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00074000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024022193625569344,20.65682601928711,0.14461298286914825,800.0 -100,200,700,800,0.0325000062584877,19.10022735595703,0.18587467074394226,800.0 -200,300,800,900,0.04759277403354645,17.978971481323242,0.22488921880722046,800.0 -300,400,900,1000,0.054005593061447144,15.063933372497559,0.2584841251373291,800.0 -400,500,1000,1100,0.03655337169766426,18.0510311126709,0.23343385756015778,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00076000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00076000.csv deleted file mode 100644 index bcbc79fdcda59a1d14e47a35bc4998f9e63d667b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00076000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027241099625825882,20.66338348388672,0.16545268893241882,800.0 -100,200,700,800,0.04145088046789169,18.405319213867188,0.22925861179828644,800.0 -200,300,800,900,0.03432688117027283,18.891096115112305,0.22914081811904907,800.0 -300,400,900,1000,0.05057145655155182,15.343414306640625,0.2660875916481018,800.0 -400,500,1000,1100,0.03291061520576477,18.561573028564453,0.24267791211605072,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00078000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00078000.csv deleted file mode 100644 index e219f4896725163ea52d5992e84a09e08237cdd7..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00078000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022168397903442383,20.917037963867188,0.1411263346672058,800.0 -100,200,700,800,0.03696296736598015,18.983440399169922,0.18052013218402863,800.0 -200,300,800,900,0.03730452433228493,19.066259384155273,0.20221933722496033,800.0 -300,400,900,1000,0.049673762172460556,16.02804183959961,0.22612138092517853,800.0 -400,500,1000,1100,0.04405733570456505,18.154897689819336,0.22802504897117615,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00080000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00080000.csv deleted file mode 100644 index 66ac5feb7d4d1908bcbf11849965187284f0a17c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00080000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02301052026450634,21.14815902709961,0.1360393762588501,800.0 -100,200,700,800,0.04006517305970192,18.140287399291992,0.1965278834104538,800.0 -200,300,800,900,0.03642890974879265,19.418485641479492,0.19675153493881226,800.0 -300,400,900,1000,0.034832872450351715,16.777565002441406,0.18309944868087769,800.0 -400,500,1000,1100,0.023617686703801155,19.747722625732422,0.18656215071678162,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00082000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00082000.csv deleted file mode 100644 index fcaba657f492909739de06c799f4bd1d89c9dd78..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00082000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02523670718073845,20.62176513671875,0.15542632341384888,800.0 -100,200,700,800,0.041885361075401306,18.648679733276367,0.18711628019809723,800.0 -200,300,800,900,0.03242404758930206,19.040260314941406,0.21288184821605682,800.0 -300,400,900,1000,0.03931339457631111,16.1864070892334,0.19766277074813843,800.0 -400,500,1000,1100,0.03045058809220791,19.239734649658203,0.20318588614463806,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00084000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00084000.csv deleted file mode 100644 index 61f039f6be0fdf65567e1f4850b30724fa1d321e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00084000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.026211638003587723,20.7037410736084,0.1383354514837265,800.0 -100,200,700,800,0.03627167269587517,18.432069778442383,0.1843893975019455,800.0 -200,300,800,900,0.03879408538341522,18.317123413085938,0.20229674875736237,800.0 -300,400,900,1000,0.04318499565124512,16.139503479003906,0.19953736662864685,800.0 -400,500,1000,1100,0.02597735822200775,19.473102569580078,0.18633899092674255,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00086000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00086000.csv deleted file mode 100644 index a4a986167ca7e58a68b3da8350fcd5c2036263f6..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00086000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.025073979049921036,20.665573120117188,0.14406123757362366,800.0 -100,200,700,800,0.04453396424651146,18.397829055786133,0.19143201410770416,800.0 -200,300,800,900,0.05059481039643288,18.668115615844727,0.22053484618663788,800.0 -300,400,900,1000,0.05232122540473938,15.649764060974121,0.20701076090335846,800.0 -400,500,1000,1100,0.0754234567284584,15.427247047424316,0.25581783056259155,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00088000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00088000.csv deleted file mode 100644 index d621ef191002abd763140f441a0b901242823ffc..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00088000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02308327704668045,20.542133331298828,0.1466791182756424,800.0 -100,200,700,800,0.04261934757232666,18.35080909729004,0.19679862260818481,800.0 -200,300,800,900,0.04030820354819298,18.671602249145508,0.22441081702709198,800.0 -300,400,900,1000,0.052965980023145676,15.641657829284668,0.23723818361759186,800.0 -400,500,1000,1100,0.04217488318681717,17.42984390258789,0.25485557317733765,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00090000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00090000.csv deleted file mode 100644 index 755f2c1a0e4be1fe2d578a59800240d0e0bcdd85..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00090000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021666033193469048,20.956438064575195,0.131177619099617,800.0 -100,200,700,800,0.04850400984287262,17.317584991455078,0.2119615226984024,800.0 -200,300,800,900,0.038719695061445236,18.900754928588867,0.20219212770462036,800.0 -300,400,900,1000,0.0538768395781517,15.782942771911621,0.25282371044158936,800.0 -400,500,1000,1100,0.05044126883149147,17.964759826660156,0.24552467465400696,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00092000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00092000.csv deleted file mode 100644 index 1f2ffde48056b6cf8eb83ff299ffa055ba90c26b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00092000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02305508591234684,21.23516845703125,0.13151033222675323,800.0 -100,200,700,800,0.038203638046979904,18.692123413085938,0.18867088854312897,800.0 -200,300,800,900,0.04346520081162453,18.706584930419922,0.2111947387456894,800.0 -300,400,900,1000,0.040818434208631516,16.543153762817383,0.20287124812602997,800.0 -400,500,1000,1100,0.030177760869264603,18.81303596496582,0.20149292051792145,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00094000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00094000.csv deleted file mode 100644 index abfdaaee4854491ed8ccb3af85f23446fd66880a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00094000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022371865808963776,21.250282287597656,0.13779447972774506,800.0 -100,200,700,800,0.03332579880952835,19.325471878051758,0.18599244952201843,800.0 -200,300,800,900,0.028140664100646973,19.844802856445312,0.1943657547235489,800.0 -300,400,900,1000,0.042786065489053726,16.360885620117188,0.20441186428070068,800.0 -400,500,1000,1100,0.030674779787659645,19.229074478149414,0.20792953670024872,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00096000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00096000.csv deleted file mode 100644 index 56c0b3b17bc734f26f18a1e8d22c6c49f1f99b01..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00096000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019231492653489113,21.483549118041992,0.12680858373641968,800.0 -100,200,700,800,0.04453633353114128,18.02498435974121,0.1970946192741394,800.0 -200,300,800,900,0.06188827380537987,17.353944778442383,0.2381657212972641,800.0 -300,400,900,1000,0.05070972070097923,15.682438850402832,0.22798055410385132,800.0 -400,500,1000,1100,0.04256841167807579,16.781408309936523,0.23515544831752777,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00098000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00098000.csv deleted file mode 100644 index d6116c8ba6b47e1f2762534cced03dd84fa6c78c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00098000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02289007604122162,20.697328567504883,0.13835059106349945,800.0 -100,200,700,800,0.0390879362821579,18.82006072998047,0.18279863893985748,800.0 -200,300,800,900,0.04850509762763977,17.76909828186035,0.22364117205142975,800.0 -300,400,900,1000,0.054596032947301865,15.452424049377441,0.23383910953998566,800.0 -400,500,1000,1100,0.03701554611325264,18.192264556884766,0.22269409894943237,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00100000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00100000.csv deleted file mode 100644 index ce23c77ae9bdd069a972250bc4c944c4a066e95e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00100000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02072089910507202,21.110685348510742,0.1366201937198639,800.0 -100,200,700,800,0.040643129497766495,18.710689544677734,0.19397060573101044,800.0 -200,300,800,900,0.04106919839978218,18.066650390625,0.2122102677822113,800.0 -300,400,900,1000,0.05937686935067177,15.5961332321167,0.2318459302186966,800.0 -400,500,1000,1100,0.057903166860342026,16.534137725830078,0.2552681863307953,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00102000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00102000.csv deleted file mode 100644 index 26eb5c944a839eb1659e2425587c9d46bf1c8e80..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00102000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019750451669096947,21.191295623779297,0.13729143142700195,800.0 -100,200,700,800,0.03387273848056793,19.240755081176758,0.18454311788082123,800.0 -200,300,800,900,0.037451356649398804,18.80799102783203,0.20805047452449799,800.0 -300,400,900,1000,0.047302208840847015,15.715044975280762,0.22208887338638306,800.0 -400,500,1000,1100,0.03896826133131981,17.501413345336914,0.22693778574466705,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00104000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00104000.csv deleted file mode 100644 index 5ecdeae0e399a88ad872a4b16df7361565a30927..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00104000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020702438428997993,21.343042373657227,0.12977169454097748,800.0 -100,200,700,800,0.03584573417901993,18.81915855407715,0.18042896687984467,800.0 -200,300,800,900,0.06103435531258583,17.612445831298828,0.2418132722377777,800.0 -300,400,900,1000,0.06282636523246765,15.254141807556152,0.2528204619884491,800.0 -400,500,1000,1100,0.03004080057144165,19.024246215820312,0.21153488755226135,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00106000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00106000.csv deleted file mode 100644 index 652da0e0d182b28b47f1fb5dda7106a6cb7f128e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00106000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027041522786021233,20.283493041992188,0.14510707557201385,800.0 -100,200,700,800,0.0553734116256237,17.63646697998047,0.21316099166870117,800.0 -200,300,800,900,0.042433127760887146,18.200088500976562,0.21338985860347748,800.0 -300,400,900,1000,0.05289529263973236,15.634913444519043,0.21022474765777588,800.0 -400,500,1000,1100,0.06169138103723526,16.317760467529297,0.26732757687568665,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00108000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00108000.csv deleted file mode 100644 index 9d2e49ead72e6bc4b6a283583b0e4946b8750e45..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00108000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024201173335313797,20.862354278564453,0.16129153966903687,800.0 -100,200,700,800,0.04253597557544708,17.889095306396484,0.2584497332572937,800.0 -200,300,800,900,0.052136268466711044,16.92841339111328,0.3024468421936035,800.0 -300,400,900,1000,0.06225172430276871,14.037215232849121,0.3384677767753601,800.0 -400,500,1000,1100,0.041952576488256454,17.227497100830078,0.29469549655914307,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00110000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00110000.csv deleted file mode 100644 index 7c94537813cbf9c4fab02b7516805476bc68b63a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00110000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024423157796263695,21.001218795776367,0.12475531548261642,800.0 -100,200,700,800,0.049867477267980576,18.45109748840332,0.1956801414489746,800.0 -200,300,800,900,0.04534006118774414,17.566686630249023,0.2307731658220291,800.0 -300,400,900,1000,0.049322959035634995,16.0767822265625,0.235112264752388,800.0 -400,500,1000,1100,0.03957132250070572,17.60321617126465,0.24778610467910767,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00112000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00112000.csv deleted file mode 100644 index b947f317408844f1ed3d295554645637115f5f3b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00112000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019393524155020714,21.5957088470459,0.12351259589195251,800.0 -100,200,700,800,0.034453459084033966,18.632001876831055,0.19121204316616058,800.0 -200,300,800,900,0.038392625749111176,18.016489028930664,0.216962531208992,800.0 -300,400,900,1000,0.04265330359339714,16.417072296142578,0.19366256892681122,800.0 -400,500,1000,1100,0.03994175046682358,17.731605529785156,0.22382067143917084,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00114000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00114000.csv deleted file mode 100644 index 82445d21d9520caa45a8a23cbfed1064079d931c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00114000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02444695495069027,20.708528518676758,0.14391912519931793,800.0 -100,200,700,800,0.03159237653017044,19.004980087280273,0.20065312087535858,800.0 -200,300,800,900,0.04050338640809059,17.82425308227539,0.22941502928733826,800.0 -300,400,900,1000,0.049505457282066345,15.897372245788574,0.2303624153137207,800.0 -400,500,1000,1100,0.026927422732114792,19.45785903930664,0.20679044723510742,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00116000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00116000.csv deleted file mode 100644 index 9edaad9d3334c7dd0c4302b88559cb97d954ef4e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00116000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01849045231938362,21.451902389526367,0.11988172680139542,800.0 -100,200,700,800,0.035384003072977066,19.121841430664062,0.17740625143051147,800.0 -200,300,800,900,0.035513125360012054,19.368202209472656,0.19637973606586456,800.0 -300,400,900,1000,0.04437476769089699,16.724321365356445,0.20204365253448486,800.0 -400,500,1000,1100,0.035473842173814774,18.229413986206055,0.21148812770843506,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00118000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00118000.csv deleted file mode 100644 index 9907072c3300149b264abc30744812337ad87a43..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00118000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02544592134654522,20.124629974365234,0.14472703635692596,800.0 -100,200,700,800,0.04264712706208229,17.67931365966797,0.21026282012462616,800.0 -200,300,800,900,0.037867702543735504,18.74469757080078,0.21688656508922577,800.0 -300,400,900,1000,0.05089452862739563,15.272482872009277,0.23756904900074005,800.0 -400,500,1000,1100,0.039386115968227386,17.89464569091797,0.2279253453016281,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00120000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00120000.csv deleted file mode 100644 index e0a3194953c448a5b5f657ddce53e4c12650ba11..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00120000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022444650530815125,20.819929122924805,0.1379595696926117,800.0 -100,200,700,800,0.04331352189183235,17.766517639160156,0.2150406837463379,800.0 -200,300,800,900,0.0390772707760334,18.202207565307617,0.23138952255249023,800.0 -300,400,900,1000,0.04574139416217804,15.7650728225708,0.2315729558467865,800.0 -400,500,1000,1100,0.03043423406779766,18.231382369995117,0.2173611968755722,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00122000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00122000.csv deleted file mode 100644 index 360322699c1e8d408ea4cb9d89b2430be7c785f7..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00122000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.023785291239619255,20.493574142456055,0.13852830231189728,800.0 -100,200,700,800,0.04570093750953674,17.673181533813477,0.21388061344623566,800.0 -200,300,800,900,0.06422556191682816,16.56566619873047,0.2709600627422333,800.0 -300,400,900,1000,0.05355772376060486,15.008155822753906,0.2550201714038849,800.0 -400,500,1000,1100,0.038875628262758255,17.290935516357422,0.24606740474700928,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00124000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00124000.csv deleted file mode 100644 index 338c6f6f1412c815ee0575890c392b6922d854db..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00124000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019737262278795242,20.825603485107422,0.13700838387012482,800.0 -100,200,700,800,0.04455946385860443,17.906757354736328,0.20101672410964966,800.0 -200,300,800,900,0.08276070654392242,15.942787170410156,0.2750447392463684,800.0 -300,400,900,1000,0.06556285917758942,14.843663215637207,0.24904434382915497,800.0 -400,500,1000,1100,0.07027267664670944,15.447126388549805,0.27325379848480225,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00126000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00126000.csv deleted file mode 100644 index d46e6f519bbbcfcc58ad87792681601ca8a648bb..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00126000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0203302763402462,21.520048141479492,0.11543164402246475,800.0 -100,200,700,800,0.03223293274641037,19.162677764892578,0.17386215925216675,800.0 -200,300,800,900,0.0363495908677578,18.884628295898438,0.199305459856987,800.0 -300,400,900,1000,0.04085622355341911,16.08881187438965,0.19266964495182037,800.0 -400,500,1000,1100,0.04144706204533577,18.441028594970703,0.22304971516132355,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00128000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00128000.csv deleted file mode 100644 index a0474b49affcd2c428a217b411246fb8248bf3fe..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00128000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02353612333536148,21.051050186157227,0.12959259748458862,800.0 -100,200,700,800,0.0411892905831337,18.901588439941406,0.18934956192970276,800.0 -200,300,800,900,0.03628597408533096,18.928213119506836,0.20742343366146088,800.0 -300,400,900,1000,0.03729148581624031,16.773832321166992,0.1829824447631836,800.0 -400,500,1000,1100,0.02859947644174099,19.25558090209961,0.2096448391675949,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00130000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00130000.csv deleted file mode 100644 index aa674e446ad5effecf5b1cf53c7255d0e17901cb..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00130000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.028416557237505913,20.40859031677246,0.14631500840187073,800.0 -100,200,700,800,0.04174340143799782,17.619796752929688,0.19615116715431213,800.0 -200,300,800,900,0.03512674197554588,18.6225528717041,0.21429216861724854,800.0 -300,400,900,1000,0.0469435453414917,15.134913444519043,0.21730583906173706,800.0 -400,500,1000,1100,0.03409091755747795,18.492481231689453,0.22742196917533875,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00132000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00132000.csv deleted file mode 100644 index ecd654421f3c54c01c4a2955d0ba5df3dbece026..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00132000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019546059891581535,21.2681884765625,0.12058841437101364,800.0 -100,200,700,800,0.032943710684776306,18.109922409057617,0.18870975077152252,800.0 -200,300,800,900,0.03528036177158356,19.03071403503418,0.2180357426404953,800.0 -300,400,900,1000,0.04658868908882141,16.065732955932617,0.22281430661678314,800.0 -400,500,1000,1100,0.03072257526218891,18.826560974121094,0.203327938914299,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00134000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00134000.csv deleted file mode 100644 index e7e6fb4cbdfdce19df85f242df9ff28d07cff9a8..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00134000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022957220673561096,20.51972007751465,0.14204424619674683,800.0 -100,200,700,800,0.035051096230745316,18.493654251098633,0.19244788587093353,800.0 -200,300,800,900,0.0654136911034584,16.620502471923828,0.2643737494945526,800.0 -300,400,900,1000,0.06706655770540237,13.734515190124512,0.3153002858161926,800.0 -400,500,1000,1100,0.08627443015575409,14.00048828125,0.3619851768016815,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00136000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00136000.csv deleted file mode 100644 index 82473b49b929a50b0c195473b10309cfc8624c6a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00136000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02556002326309681,20.476625442504883,0.14866790175437927,800.0 -100,200,700,800,0.042587343603372574,18.57271385192871,0.19825994968414307,800.0 -200,300,800,900,0.04553584009408951,17.72321128845215,0.22838816046714783,800.0 -300,400,900,1000,0.04895840957760811,16.02972984313965,0.22485344111919403,800.0 -400,500,1000,1100,0.04970831796526909,17.016284942626953,0.23766006529331207,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00138000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00138000.csv deleted file mode 100644 index 548c78247eaa62e2d67f28fb71548347568749e6..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00138000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01852991431951523,21.173921585083008,0.12617628276348114,800.0 -100,200,700,800,0.0339324027299881,18.407907485961914,0.20882000029087067,800.0 -200,300,800,900,0.04469023272395134,17.660058975219727,0.25500550866127014,800.0 -300,400,900,1000,0.06384105235338211,14.755624771118164,0.27977922558784485,800.0 -400,500,1000,1100,0.04381512105464935,17.024141311645508,0.25819140672683716,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00140000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00140000.csv deleted file mode 100644 index 7c4fd5bb46e50763ec9451b226b987d0f136bb5b..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00140000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017759090289473534,21.360549926757812,0.1352950632572174,800.0 -100,200,700,800,0.03653841093182564,19.372722625732422,0.18254947662353516,800.0 -200,300,800,900,0.03326354920864105,19.259319305419922,0.18190838396549225,800.0 -300,400,900,1000,0.04080865532159805,16.726890563964844,0.18896587193012238,800.0 -400,500,1000,1100,0.036861468106508255,18.77068519592285,0.2061426192522049,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00142000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00142000.csv deleted file mode 100644 index 17441e5a05f60eccaf59a174a4c392e0db2b92b8..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00142000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017414281144738197,21.302234649658203,0.1149839237332344,800.0 -100,200,700,800,0.05567002668976784,17.371591567993164,0.22577987611293793,800.0 -200,300,800,900,0.036447104066610336,18.84510040283203,0.22186344861984253,800.0 -300,400,900,1000,0.0481528677046299,16.138490676879883,0.22198204696178436,800.0 -400,500,1000,1100,0.05713598430156708,15.675457000732422,0.2620561718940735,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00144000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00144000.csv deleted file mode 100644 index 1cd9bfbf4d6b364dfedaaafb14b09919fc79508d..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00144000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.020410846918821335,21.480127334594727,0.11910812556743622,800.0 -100,200,700,800,0.03965027630329132,19.33711814880371,0.17741389572620392,800.0 -200,300,800,900,0.03432612866163254,18.93834686279297,0.20025452971458435,800.0 -300,400,900,1000,0.03031443990767002,17.530536651611328,0.16480106115341187,800.0 -400,500,1000,1100,0.03147323429584503,18.980623245239258,0.20294107496738434,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00146000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00146000.csv deleted file mode 100644 index 22cb89877d7b7c0c2802ac273a2485e4fd16121d..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00146000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015733608976006508,21.723806381225586,0.10852321982383728,800.0 -100,200,700,800,0.040301889181137085,19.022554397583008,0.18099109828472137,800.0 -200,300,800,900,0.04464827477931976,18.441682815551758,0.22459229826927185,800.0 -300,400,900,1000,0.04391159862279892,16.301651000976562,0.20823371410369873,800.0 -400,500,1000,1100,0.03772883489727974,18.554147720336914,0.23042501509189606,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00148000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00148000.csv deleted file mode 100644 index cc69278469113a2fecae516360d418178c5f2ddb..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00148000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019417397677898407,21.129621505737305,0.14486348628997803,800.0 -100,200,700,800,0.03302516043186188,18.49713706970215,0.19441215693950653,800.0 -200,300,800,900,0.03153199702501297,19.18259048461914,0.20634612441062927,800.0 -300,400,900,1000,0.046517014503479004,16.375144958496094,0.22295904159545898,800.0 -400,500,1000,1100,0.031988710165023804,18.333345413208008,0.21102339029312134,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00150000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00150000.csv deleted file mode 100644 index c53ff9f1af4acbfea60bf4bcc2d2c6cea9d32d07..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00150000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02070889063179493,21.373544692993164,0.12723995745182037,800.0 -100,200,700,800,0.029915787279605865,19.519493103027344,0.17058372497558594,800.0 -200,300,800,900,0.029997410252690315,19.52944564819336,0.18627920746803284,800.0 -300,400,900,1000,0.03694554418325424,17.2529239654541,0.18975841999053955,800.0 -400,500,1000,1100,0.02526063285768032,19.219453811645508,0.20123466849327087,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00152000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00152000.csv deleted file mode 100644 index 942f0e857bae16cf04b638a2370af52f8086ba1a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00152000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.027014270424842834,20.564857482910156,0.15640312433242798,800.0 -100,200,700,800,0.03307762369513512,19.294456481933594,0.18410418927669525,800.0 -200,300,800,900,0.034505173563957214,18.141904830932617,0.20024743676185608,800.0 -300,400,900,1000,0.037225473672151566,16.512577056884766,0.18976745009422302,800.0 -400,500,1000,1100,0.02632938139140606,19.36969757080078,0.20241470634937286,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00154000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00154000.csv deleted file mode 100644 index 0aaeb2e3ce2aeffc0d940ca8bd3968d0cb55732a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00154000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.031782690435647964,20.342632293701172,0.14914445579051971,800.0 -100,200,700,800,0.04400290548801422,18.703927993774414,0.19597825407981873,800.0 -200,300,800,900,0.04019692540168762,18.14044189453125,0.22294887900352478,800.0 -300,400,900,1000,0.056036341935396194,15.169659614562988,0.25511690974235535,800.0 -400,500,1000,1100,0.04185358062386513,17.52554702758789,0.25944656133651733,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00156000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00156000.csv deleted file mode 100644 index 6b813b06f9f63d1100c82543965dcb38304993a3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00156000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016235124319791794,21.900684356689453,0.10986334085464478,800.0 -100,200,700,800,0.029487168416380882,19.533945083618164,0.17060421407222748,800.0 -200,300,800,900,0.03857189789414406,18.89946937561035,0.2129199206829071,800.0 -300,400,900,1000,0.04393772780895233,16.594106674194336,0.20048698782920837,800.0 -400,500,1000,1100,0.032565634697675705,18.4209041595459,0.207418292760849,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00158000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00158000.csv deleted file mode 100644 index 0fd78fbce8ce56edb7d6772a54761aa19dc58c29..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00158000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01866810768842697,21.448644638061523,0.11828429251909256,800.0 -100,200,700,800,0.02763058990240097,19.357494354248047,0.1576118916273117,800.0 -200,300,800,900,0.030685115605592728,19.360563278198242,0.18795259296894073,800.0 -300,400,900,1000,0.03751666843891144,16.624849319458008,0.18666790425777435,800.0 -400,500,1000,1100,0.026398269459605217,19.143503189086914,0.19945192337036133,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00160000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00160000.csv deleted file mode 100644 index 36546b7d9e47cdb62c7246c9aa153526298b9867..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00160000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015353161841630936,22.032514572143555,0.10390972346067429,800.0 -100,200,700,800,0.03902828320860863,19.337989807128906,0.16600829362869263,800.0 -200,300,800,900,0.03395133092999458,19.41045379638672,0.18814456462860107,800.0 -300,400,900,1000,0.03564266487956047,17.449325561523438,0.18665660917758942,800.0 -400,500,1000,1100,0.023830270394682884,19.326505661010742,0.19081905484199524,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00162000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00162000.csv deleted file mode 100644 index fea66c0a1f380c7c579bf6d3ee3738bfdd8c9760..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00162000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.022196343168616295,21.27136993408203,0.1277405321598053,800.0 -100,200,700,800,0.03758712857961655,18.9946346282959,0.17881645262241364,800.0 -200,300,800,900,0.04114202409982681,18.573780059814453,0.20763541758060455,800.0 -300,400,900,1000,0.04411069303750992,16.36833953857422,0.204060897231102,800.0 -400,500,1000,1100,0.03662614896893501,17.70628547668457,0.21082212030887604,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00164000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00164000.csv deleted file mode 100644 index 3980ea4c34dc15e867ef32de8c3f2da3f63d0a5c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00164000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02217293530702591,20.82349395751953,0.1320718675851822,800.0 -100,200,700,800,0.03413039818406105,18.529857635498047,0.1860688328742981,800.0 -200,300,800,900,0.04366572946310043,17.844078063964844,0.22592681646347046,800.0 -300,400,900,1000,0.04722631350159645,15.530686378479004,0.21947742998600006,800.0 -400,500,1000,1100,0.03504855930805206,18.092144012451172,0.22164711356163025,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00166000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00166000.csv deleted file mode 100644 index 0d003bba99afc92656623c58d04dd691df308200..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00166000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0192184466868639,20.580001831054688,0.14537423849105835,800.0 -100,200,700,800,0.03411217778921127,19.064184188842773,0.18781867623329163,800.0 -200,300,800,900,0.04147326946258545,18.609193801879883,0.21843267977237701,800.0 -300,400,900,1000,0.05270640552043915,15.464191436767578,0.22670979797840118,800.0 -400,500,1000,1100,0.05050274357199669,17.80054473876953,0.22805896401405334,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00168000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00168000.csv deleted file mode 100644 index 03a49f635facb0849745aaab12a97cddad860b1a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00168000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.0236700177192688,20.233022689819336,0.12876996397972107,800.0 -100,200,700,800,0.036919549107551575,18.615371704101562,0.18936489522457123,800.0 -200,300,800,900,0.049627579748630524,17.752103805541992,0.23477642238140106,800.0 -300,400,900,1000,0.0465879961848259,15.932975769042969,0.23193033039569855,800.0 -400,500,1000,1100,0.029939943924546242,18.98981475830078,0.2287488430738449,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00170000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00170000.csv deleted file mode 100644 index 8f2a8f4d59b4ccdbb449e364c295391cc64cc9ac..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00170000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.017694037407636642,21.474443435668945,0.10777170956134796,800.0 -100,200,700,800,0.03278152644634247,19.778240203857422,0.1590503752231598,800.0 -200,300,800,900,0.036873847246170044,19.116846084594727,0.20260387659072876,800.0 -300,400,900,1000,0.04565555974841118,16.482206344604492,0.22812968492507935,800.0 -400,500,1000,1100,0.027891242876648903,19.49457550048828,0.1957874298095703,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00172000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00172000.csv deleted file mode 100644 index 869579a4fa9c1c5d9e5eeeb4a0a2a1a23fc81ec3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00172000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.019561484456062317,21.20246124267578,0.1130690947175026,800.0 -100,200,700,800,0.030950861051678658,19.519336700439453,0.16221679747104645,800.0 -200,300,800,900,0.04345979541540146,19.104454040527344,0.2125900685787201,800.0 -300,400,900,1000,0.041732512414455414,16.716096878051758,0.1943313628435135,800.0 -400,500,1000,1100,0.034251369535923004,18.73080062866211,0.21567466855049133,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00174000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00174000.csv deleted file mode 100644 index 5e63aa112c4b7f65d5576871613c42cc79b7031a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00174000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02192540094256401,20.777585983276367,0.14788071811199188,800.0 -100,200,700,800,0.04215094447135925,18.676958084106445,0.19088798761367798,800.0 -200,300,800,900,0.06375683099031448,17.31702995300293,0.23445163667201996,800.0 -300,400,900,1000,0.06346652656793594,15.652734756469727,0.23126526176929474,800.0 -400,500,1000,1100,0.058601751923561096,17.237306594848633,0.24120935797691345,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00176000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00176000.csv deleted file mode 100644 index 57820744394bdd2b386cad3f692e8f8a58b0da8c..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00176000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.014467933215200901,21.840099334716797,0.10441771149635315,800.0 -100,200,700,800,0.04150281846523285,18.618330001831055,0.1866922378540039,800.0 -200,300,800,900,0.036614514887332916,18.73453140258789,0.19586201012134552,800.0 -300,400,900,1000,0.046580761671066284,15.812849044799805,0.1971929669380188,800.0 -400,500,1000,1100,0.034510139375925064,18.112363815307617,0.19990013539791107,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00178000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00178000.csv deleted file mode 100644 index 3620f2bf4cd6520dbc0db8867d1f5beae87c9805..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00178000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.024547934532165527,20.57111358642578,0.14251112937927246,800.0 -100,200,700,800,0.029681555926799774,19.52387237548828,0.16924186050891876,800.0 -200,300,800,900,0.028522344306111336,19.61078453063965,0.1900935024023056,800.0 -300,400,900,1000,0.0412166491150856,16.39310073852539,0.19098953902721405,800.0 -400,500,1000,1100,0.022203557193279266,19.838844299316406,0.18991802632808685,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00180000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00180000.csv deleted file mode 100644 index 566173ad4b1793eb72ece78725e2e97cf92fe071..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00180000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021499106660485268,20.996410369873047,0.1427019238471985,800.0 -100,200,700,800,0.03870199993252754,18.685523986816406,0.18725815415382385,800.0 -200,300,800,900,0.04449796676635742,17.771896362304688,0.22949768602848053,800.0 -300,400,900,1000,0.04980384185910225,15.664192199707031,0.2256464809179306,800.0 -400,500,1000,1100,0.04255693033337593,17.564586639404297,0.22170883417129517,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00182000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00182000.csv deleted file mode 100644 index a08f7d1cf252fd27d47f1fb401d2b9850c32378e..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00182000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.02635602280497551,20.575273513793945,0.1400618553161621,800.0 -100,200,700,800,0.05065770447254181,17.950969696044922,0.1907120645046234,800.0 -200,300,800,900,0.05062803253531456,17.490158081054688,0.23862792551517487,800.0 -300,400,900,1000,0.04502551257610321,15.737669944763184,0.21642141044139862,800.0 -400,500,1000,1100,0.056453317403793335,16.521610260009766,0.24930982291698456,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00184000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00184000.csv deleted file mode 100644 index 97c25b57f1e0524605e953535bf2b8916049fc90..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00184000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021899079903960228,21.035615921020508,0.13667987287044525,800.0 -100,200,700,800,0.04212385043501854,18.493810653686523,0.19102995097637177,800.0 -200,300,800,900,0.03547540307044983,19.349794387817383,0.20522144436836243,800.0 -300,400,900,1000,0.04449762776494026,16.295730590820312,0.22024206817150116,800.0 -400,500,1000,1100,0.03150784596800804,18.292985916137695,0.22452770173549652,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00186000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00186000.csv deleted file mode 100644 index ea600c546645dc0e5796b195d8960231d0fa6d9a..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00186000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016710752621293068,21.529579162597656,0.11124172061681747,800.0 -100,200,700,800,0.03954996168613434,19.047073364257812,0.17851769924163818,800.0 -200,300,800,900,0.04982394352555275,18.690336227416992,0.2124084234237671,800.0 -300,400,900,1000,0.05257053300738335,15.573005676269531,0.243004709482193,800.0 -400,500,1000,1100,0.09181003272533417,16.427562713623047,0.26848873496055603,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00188000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00188000.csv deleted file mode 100644 index fb11ea236765cd5880417b3b755450d626b341cc..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00188000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016777418553829193,21.736328125,0.1089155450463295,800.0 -100,200,700,800,0.038909848779439926,18.92121696472168,0.18438231945037842,800.0 -200,300,800,900,0.03584309294819832,18.79851722717285,0.2074844241142273,800.0 -300,400,900,1000,0.04420402646064758,15.592975616455078,0.23284441232681274,800.0 -400,500,1000,1100,0.03363476321101189,18.417699813842773,0.2228223830461502,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00190000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00190000.csv deleted file mode 100644 index 90262f039e6e29ba1f6434b1ca4240102ed7daf0..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00190000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.01661394163966179,21.466079711914062,0.11225967109203339,800.0 -100,200,700,800,0.03691750392317772,19.044456481933594,0.17276176810264587,800.0 -200,300,800,900,0.03277428448200226,19.206695556640625,0.18748335540294647,800.0 -300,400,900,1000,0.03615687042474747,17.39055061340332,0.17077018320560455,800.0 -400,500,1000,1100,0.02193734236061573,19.45705795288086,0.17757835984230042,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00192000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00192000.csv deleted file mode 100644 index d4ed21ea7bbd13b12c40b4968109f3b09f4a2968..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00192000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.021133730188012123,20.989484786987305,0.1286303997039795,800.0 -100,200,700,800,0.04691638797521591,18.559646606445312,0.18442142009735107,800.0 -200,300,800,900,0.037980128079652786,18.97903060913086,0.21525847911834717,800.0 -300,400,900,1000,0.037060510367155075,16.922386169433594,0.19662964344024658,800.0 -400,500,1000,1100,0.033570967614650726,18.860393524169922,0.2196812629699707,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00194000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00194000.csv deleted file mode 100644 index 6d93cf15f2aee3e133441b9cd427589ffa1a6973..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00194000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.018724653869867325,21.260149002075195,0.12387976050376892,800.0 -100,200,700,800,0.03664426505565643,18.764467239379883,0.17109178006649017,800.0 -200,300,800,900,0.05004531517624855,17.87690544128418,0.22389350831508636,800.0 -300,400,900,1000,0.04459044337272644,16.4213924407959,0.19356082379817963,800.0 -400,500,1000,1100,0.05655481666326523,16.796960830688477,0.22482052445411682,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00196000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00196000.csv deleted file mode 100644 index 6bfdce8656d24da819994618fa11088f79def1d7..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00196000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.016272835433483124,21.6944637298584,0.11075278371572495,800.0 -100,200,700,800,0.03276675194501877,19.696060180664062,0.16426286101341248,800.0 -200,300,800,900,0.03446605056524277,19.48448944091797,0.18629883229732513,800.0 -300,400,900,1000,0.03031475469470024,17.46364402770996,0.16146092116832733,800.0 -400,500,1000,1100,0.026261592283844948,19.688072204589844,0.19375106692314148,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00198000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00198000.csv deleted file mode 100644 index 706feb9b508617446f76571f6ac61fb81a093b64..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00198000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015882859006524086,21.833696365356445,0.10801839828491211,800.0 -100,200,700,800,0.03801101818680763,19.10906982421875,0.17746245861053467,800.0 -200,300,800,900,0.03359970077872276,18.83147621154785,0.19848217070102692,800.0 -300,400,900,1000,0.03785400092601776,17.11301040649414,0.17382943630218506,800.0 -400,500,1000,1100,0.031266678124666214,18.30303192138672,0.20402640104293823,800.0 diff --git a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00200000.csv b/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00200000.csv deleted file mode 100644 index 033f3f8da603d1c8ddc9c934c2b78b20426623b3..0000000000000000000000000000000000000000 --- a/train_dememwm_unified_dense_memory_ddp_selection_geometry_8h200_bs16_global128_200k/wandb/offline-run-20260714_201232-k0w2g95q/files/metrics/segment_metrics_step00200000.csv +++ /dev/null @@ -1,6 +0,0 @@ -generation_start,generation_end,absolute_start,absolute_end,mse,psnr,lpips,count -0,100,600,700,0.015244786627590656,21.687143325805664,0.11389598995447159,800.0 -100,200,700,800,0.0283520445227623,19.729917526245117,0.16710418462753296,800.0 -200,300,800,900,0.027918512001633644,19.769704818725586,0.17563065886497498,800.0 -300,400,900,1000,0.03124520741403103,17.467756271362305,0.16541241109371185,800.0 -400,500,1000,1100,0.024286948144435883,19.633878707885742,0.18673183023929596,800.0