Add files using upload-large-folder tool

checkpoints/cf-cr4-lr1e4-restart-10k/config.yaml

@@ -0,0 +1,469 @@
+checkpoint:
+  broadcast_via_filesystem: 'False'
+  dcp_allow_mismatched_size: 'False'
+  dcp_async_mode_enabled: 'False'
+  jit:
+    device: cuda
+    dtype: bfloat16
+    enabled: 'False'
+    input_shape: null
+    strict: 'True'
+  keys_not_to_resume: []
+  load_ema_to_reg: 'False'
+  load_from_object_store:
+    bucket: ''
+    credentials: ''
+    enabled: 'False'
+  load_path: ''
+  load_training_state: 'False'
+  only_load_scheduler_state: 'False'
+  override_optimizer_lr: 'False'
+  save_iter: '125'
+  save_to_object_store:
+    bucket: ''
+    credentials: ''
+    enabled: 'False'
+  strict_resume: 'False'
+  type:
+    _target_: <class 'rcm.checkpointers.dcp_distill.DistributedCheckpointer_Distill'>
+    callbacks: null
+    disable_async: 'False'
+  verbose: 'True'
+dataloader_train:
+  _target_: <function create_dataloader at 0x7f86bd0972e0>
+  batch_size: '1'
+  max_samples: '0'
+  num_workers: '1'
+  prefetch_factor: '2'
+  sample_offset: '19750'
+  shard_shuffle_buffer: '0'
+  shuffle_buffer: '0'
+  tar_path_pattern: /scratch/wg25/video-compress/datasets/causal_forcing_c3-3_rollout_81920train_256val_seed0/parts/train/part-*/shard-00000.tar
+  uniform_shard_samples: '128'
+dataloader_val:
+  _target_: <class 'torch.utils.data.dataloader.DataLoader'>
+  batch_sampler: null
+  batch_size: '1'
+  collate_fn: null
+  dataset: <function <lambda> at 0x7f86bd0316c0>
+  drop_last: 'False'
+  generator: null
+  in_order: 'True'
+  multiprocessing_context: null
+  num_workers: '0'
+  persistent_workers: 'False'
+  pin_memory: 'False'
+  pin_memory_device: ''
+  prefetch_factor: null
+  sampler: null
+  shuffle: null
+  timeout: '0'
+  worker_init_fn: null
+defaults:
+- _self_
+- trainer: standard
+- data_train: dummy
+- data_val: dummy
+- optimizer: fusedadamw
+- scheduler: lambdalinear
+- callbacks: basic
+- checkpoint: local
+- ckpt_type: dcp
+- model: fsdp_t2v_distill_rcm
+- net: null
+- net_compressor: null
+- net_teacher: null
+- net_causal_teacher: null
+- net_bidirectional_teacher: null
+- net_fake_score: null
+- optimizer_fake_score: fusedadamw
+- conditioner: text_nodrop
+- ema: power
+- tokenizer: wan2pt1_tokenizer
+- experiment: null
+job:
+  group: future-sf-lr1e4-restart
+  name: context_compress_causal_forcing_future_sf_c3-3_81920train_cr4_packed_temporal_grid_gbs8_lr1e-4_10000it
+  project: Memory_Compression
+  wandb_mode: online
+model:
+  _recursive_: 'False'
+  _target_: <class 'rcm.models.t2v_model_causal.T2VCausalModel'>
+  config:
+    backward_timesteps:
+    - '0.9375'
+    - '0.8333333333333334'
+    - '0.625'
+    bidirectional_guidance: 5.0
+    bidirectional_teacher_ckpt: ''
+    causal_guidance: 3.0
+    causal_teacher_ckpt: ''
+    checkpoint:
+      bucket: ''
+      credentials: ''
+      enabled: false
+    chunk_t: 3
+    compressor_ckpt: ''
+    conditioner:
+      _target_: <class 'rcm.conditioner.TextConditioner'>
+      text:
+        _target_: <class 'rcm.conditioner.TextAttr'>
+        dropout_rate: '0.0'
+        empty_string_embeddings_path: negative_prompt/empty_string_umt5.pt
+        input_key:
+        - t5_text_embeddings
+        use_empty_string: 'False'
+    context_compression_ae_weight: 1.0
+    context_compression_cacheaware_branch: future
+    context_compression_causal_forcing_numerics: true
+    context_compression_compressed_context_checkpoint_mode: reentrant
+    context_compression_compressor_checkpoint_mode: default
+    context_compression_disable_compressed_context_checkpoint: false
+    context_compression_history_t: 0
+    context_compression_history_t_max: 20
+    context_compression_history_t_min: 1
+    context_compression_memory_rope_mode: packed_temporal_grid
+    context_compression_pad_sampled_history_t: false
+    context_compression_prefix_t_values:
+    - '3'
+    - '6'
+    - '9'
+    - '12'
+    - '15'
+    - '18'
+    context_compression_rollout_denoise_steps: 4
+    context_compression_rollout_detach_history: true
+    context_compression_rollout_loss_reduction: mean
+    context_compression_rollout_steps: 0
+    context_compression_sample_history_t: false
+    context_compression_sample_target_start_t: false
+    context_compression_static_fixed_noise: false
+    context_compression_static_t: -1.0
+    context_compression_static_t_values: ''
+    context_compression_t_steps: 1,15/16,5/6,5/8,0
+    context_compression_target_start_t_max: 0
+    context_compression_target_start_t_min: 0
+    context_compression_target_t: 3
+    context_from_last_step: false
+    context_from_last_step_start_chunk: 0
+    dcm_skipping_interval_steps: 1
+    dcm_total_steps: 48
+    df_timestep_strategy: random
+    ema:
+      enabled: false
+      iteration_shift: 0
+      rate: 0.1
+    fake_score_ckpt: ''
+    first_chunk_t: 3
+    fsdp_shard_size: 1
+    generation_dit_frozen: true
+    grad_clip: false
+    input_caption_key: prompts
+    input_data_key: videos
+    input_latent_key: latents
+    loss_scale: 100.0
+    loss_scale_dmd: 1.0
+    loss_scale_fake_score: 1.0
+    loss_weighting: none
+    max_simulation_steps_fake: 4
+    neg_embed_path: ''
+    net:
+      _target_: <class 'rcm.networks.wan2pt1.WanModel'>
+      cross_attn_norm: 'True'
+      dim: '1536'
+      eps: 1e-06
+      ffn_dim: '8960'
+      freq_dim: '256'
+      in_dim: '16'
+      model_type: t2v
+      num_heads: '12'
+      num_layers: '30'
+      out_dim: '16'
+      patch_size:
+      - '1'
+      - '2'
+      - '2'
+      qk_norm: 'True'
+      sac_config:
+        determinism_check: default
+        every_n_blocks: 1
+        mode: block_wise
+      text_dim: '4096'
+      text_len: '512'
+    net_bidirectional_teacher: null
+    net_causal_teacher: null
+    net_compressor:
+      _target_: <class 'rcm.networks.wan2pt1.WanMemoryCompressor'>
+      cross_attn_norm: 'True'
+      dim: '1536'
+      eps: 1e-06
+      ffn_dim: '8960'
+      freq_dim: '256'
+      in_dim: '16'
+      max_memory_tokens_per_frame: '1024'
+      memory_ratio: '0.25'
+      model_type: t2v
+      num_heads: '12'
+      num_layers: '30'
+      out_dim: '16'
+      patch_size:
+      - '1'
+      - '2'
+      - '2'
+      qk_norm: 'True'
+      sac_config:
+        determinism_check: default
+        every_n_blocks: 1
+        mode: block_wise
+      text_dim: '4096'
+      text_len: '512'
+    net_fake_score: null
+    optimizer_fake_score:
+      _target_: <function get_base_optimizer at 0x7f86c8ec25c0>
+      betas:
+      - '0.9'
+      - '0.99'
+      capturable: 'True'
+      eps: 1e-08
+      lr: '0.0001'
+      master_weights: 'True'
+      model: null
+      optim_type: fusedadam
+      weight_decay: '0.1'
+    p_D:
+      _target_: <class 'rcm.utils.timestep_utils.UniformShift'>
+      shift: '5.0'
+    p_G:
+      _target_: <class 'rcm.utils.timestep_utils.UniformShift'>
+      p_mean: '-0.8'
+      p_std: '1.6'
+      shift: '5.0'
+    precision: bfloat16
+    rectified_flow_t_scaling_factor: 1000.0
+    replayed_training: false
+    resolution: 480p
+    sample_timestep_shift: 3.0
+    sf_backward_timestep_schedules: []
+    sf_simulation_steps_per_chunk: []
+    shift3_backward: false
+    sigma_data: 1.0
+    sigma_max: 1600.0
+    state_ch: 16
+    state_t: 21
+    strict_raw_checkpoint_load: true
+    student_ckpt: /scratch/wg25/video-compress/assets/checkpoints/Causal-Forcing/chunkwise/causal_forcing.pt
+    student_update_freq: 5
+    tangent_warmup: 0
+    text_encoder_class: umT5
+    text_encoder_path: ''
+    tokenizer:
+      _target_: <class 'rcm.tokenizers.wan2pt1.Wan2pt1VAEInterface'>
+      chunk_duration: '81'
+      vae_pth: /scratch/wg25/video-compress/assets/checkpoints/Wan2.1_VAE.pth
+    training_type: cc_cacheaware_self_forcing
+model_parallel:
+  _cpu_offloading_context: null
+  async_tensor_model_parallel_allreduce: true
+  autocast_dtype: torch.float32
+  barrier_with_L1_time: true
+  batch_p2p_comm: true
+  batch_p2p_sync: true
+  bf16: false
+  context_parallel_size: 1
+  cpu_offloading: false
+  cpu_offloading_activations: true
+  cpu_offloading_double_buffering: false
+  cpu_offloading_num_layers: 0
+  cpu_offloading_weights: false
+  cross_entropy_fusion_impl: native
+  cross_entropy_loss_fusion: false
+  deallocate_pipeline_outputs: false
+  defer_embedding_wgrad_compute: false
+  delay_wgrad_compute: false
+  deterministic_mode: false
+  enable_autocast: false
+  ep_overlap_early_attn_memory_release: false
+  expert_model_parallel_size: 1
+  expert_tensor_parallel_size: 1
+  finalize_model_grads_func: null
+  fp16: false
+  grad_scale_func: null
+  grad_sync_func: null
+  gradient_accumulation_fusion: false
+  hierarchical_context_parallel_sizes: null
+  hybrid_context_parallel: false
+  max_seqlen_per_dp_cp_rank: null
+  microbatch_group_size_per_vp_stage: 1
+  moe_extended_tp: false
+  mtp_standalone: false
+  no_sync_func: null
+  num_microbatches_with_partial_activation_checkpoints: null
+  overlap_moe_expert_parallel_comm: false
+  overlap_p2p_comm: false
+  overlap_p2p_comm_warmup_flush: false
+  param_sync_func: null
+  params_dtype: torch.float32
+  perform_initialization: true
+  pipeline_dtype: null
+  pipeline_model_parallel_comm_backend: null
+  pipeline_model_parallel_size: 1
+  sequence_parallel: false
+  tensor_model_parallel_size: 1
+  timers: null
+  tp_comm_atomic_ag: false
+  tp_comm_atomic_rs: false
+  tp_comm_bootstrap_backend: nccl
+  tp_comm_bulk_dgrad: true
+  tp_comm_bulk_wgrad: true
+  tp_comm_overlap: false
+  tp_comm_overlap_ag: true
+  tp_comm_overlap_disable_fc1: false
+  tp_comm_overlap_disable_qkv: false
+  tp_comm_overlap_rs: true
+  tp_comm_overlap_rs_dgrad: false
+  tp_comm_split_ag: true
+  tp_comm_split_rs: true
+  use_cpu_initialization: false
+  use_ring_exchange_p2p: false
+  use_te_rng_tracker: false
+  variable_seq_lengths: false
+  virtual_pipeline_model_parallel_size: null
+  wgrad_deferral_limit: 0
+optimizer:
+  _target_: <function get_base_optimizer at 0x7f86c8ec25c0>
+  betas:
+  - '0.9'
+  - '0.999'
+  capturable: 'True'
+  eps: 1e-08
+  lr: '0.0001'
+  master_weights: 'True'
+  model: null
+  optim_type: fusedadam
+  weight_decay: '0.01'
+scheduler:
+  _target_: <class 'rcm.utils.lr_scheduler.LambdaWarmUpCosineScheduler'>
+  cycle_lengths:
+  - '10000'
+  f_max:
+  - '1.0'
+  f_min:
+  - '0.5'
+  f_start:
+  - 1e-06
+  verbosity_interval: '0'
+  warm_up_steps:
+  - '100'
+trainer:
+  callbacks:
+    compile_tokenizer:
+      _target_: <class 'rcm.callbacks.compile_tokenizer.CompileTokenizer'>
+      compile_after_iterations: '0'
+      dynamic: 'False'
+      enabled: 'False'
+    dataloader_speed:
+      _target_: <class 'rcm.callbacks.dataloading_monitor.DetailedDataLoadingSpeedMonitor'>
+      every_n: '5'
+      save_s3: 'False'
+      step_size: '1'
+    device_monitor:
+      _target_: <class 'rcm.callbacks.device_monitor.DeviceMonitor'>
+      every_n: '5'
+      log_memory_detail: 'True'
+      save_s3: 'False'
+      step_size: '1'
+      upload_every_n_mul: '10'
+    every_n_sample_ema:
+      _target_: <class 'rcm.callbacks.every_n_draw_distill.EveryNDrawSample_Distill'>
+      every_n: '100000000'
+      is_ema: 'True'
+      is_image: 'False'
+      is_sample: 'True'
+      n_sample_to_save: '64'
+      num_samples: '2'
+      num_sampling_step: '50'
+      run_at_start: 'False'
+      sample_fix: 'False'
+      save_s3: 'False'
+      show_all_frames: 'False'
+      step_size: '1'
+    every_n_sample_reg:
+      _target_: <class 'rcm.callbacks.every_n_draw_distill.EveryNDrawSample_Distill'>
+      every_n: '100000000'
+      is_ema: 'False'
+      is_image: 'False'
+      is_sample: 'True'
+      n_sample_to_save: '64'
+      num_samples: '2'
+      num_sampling_step: '50'
+      run_at_start: 'False'
+      sample_fix: 'False'
+      save_s3: 'False'
+      show_all_frames: 'False'
+      step_size: '1'
+    grad_clip:
+      _target_: <class 'rcm.callbacks.grad_clip.GradClip'>
+      clip_norm: '5.0'
+      force_finite: 'True'
+    heart_beat:
+      _target_: <class 'rcm.callbacks.heart_beat.HeartBeat'>
+      every_n: '10'
+      save_s3: 'False'
+      step_size: '1'
+      update_interval_in_minute: '20'
+    iter_speed:
+      _target_: <class 'rcm.callbacks.iter_speed.IterSpeed'>
+      every_n: '5'
+      hit_thres: '5'
+      save_s3: 'False'
+      save_s3_every_log_n: '10'
+    low_prec:
+      _target_: <class 'imaginaire.callbacks.low_precision.LowPrecisionCallback'>
+      config: null
+      trainer: null
+      update_iter: '1'
+    manual_gc:
+      _target_: <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>
+      every_n: '5'
+      warm_up: '5'
+    wandb:
+      _target_: <class 'rcm.callbacks.wandb_log.WandbCallback'>
+      logging_iter_multipler: '1'
+      save_logging_iter_multipler: '10'
+      save_s3: 'False'
+    wandb_10x:
+      _target_: <class 'rcm.callbacks.wandb_log.WandbCallback'>
+      logging_iter_multipler: '10'
+      save_logging_iter_multipler: '1'
+      save_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp:
+    broadcast_buffers: 'True'
+    find_unused_parameters: 'False'
+    static_graph: 'True'
+  distributed_parallelism: ddp
+  grad_accum_iter: '2'
+  grad_scaler_args:
+    enabled: 'False'
+  logging_iter: '5'
+  max_iter: '10000'
+  max_val_iter: null
+  memory_format: torch.preserve_format
+  profiling:
+    enable_memory_snapshot: 'False'
+    enable_profiling: 'False'
+    first_n_rank: '8'
+    profile_freq: '1'
+    profile_memory: 'True'
+    record_shape: 'True'
+    with_modules: 'True'
+    with_stack: 'True'
+  run_validation: 'False'
+  seed: '0'
+  timeout_period: '999999999'
+  type:
+    _target_: <class 'rcm.trainers.trainer_distillation.ImaginaireTrainer_Distill'>
+    config: null
+  validation_iter: '100'