Add c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000 (milestone, cosine 2e-4 -> 5e-5)

.gitattributes

@@ -95,3 +95,5 @@ c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000007000/model/.metadata filt
 c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000007000/model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
 c3-3/cosine_2e-4_to5e-5_clip5/packed_temporal_grid/iter_000008000/model/.metadata filter=lfs diff=lfs merge=lfs -text
 c3-3/cosine_2e-4_to5e-5_clip5/packed_temporal_grid/iter_000008000/model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/model/.metadata filter=lfs diff=lfs merge=lfs -text
+c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/README.md

@@ -0,0 +1,23 @@
+# c3-3 / source_aligned / step 8000
+
+- **Model family**: `c3-3`
+- **Memory RoPE mode**: `source_aligned`
+- **Optimizer step**: 8000
+- **Sample exposure**: 64000 (step x global batch 8)
+- **W&B run ID**: `jhnefd4q`
+- **Code commit**: `44f4c8b11f671e811f896539c5faa96db77a078d`
+- **Dataset manifest SHA-256**: `488feab621a47d5f5b77e1fb6547b24d64912cba66b7d6cababa53c1d4c8dd78`
+- **Learning rate**: 0.0002 (peak; cosine to 5e-05 = x0.25, 100-step warmup, cycle 10000)
+- **Gradient clipping**: enabled=true, clip_norm=5.0
+- **Generation geometry**: first_chunk_t=3, chunk_t=3
+- **Contents**: model/inference state only (8 DCP shards + `.metadata`); optimizer and scheduler state excluded
+
+## Fixed-noise validation
+
+No fixed-noise validation result exists for this exact step.
+
+## Shard layout
+
+All tensors are consolidated into `__0_0.distcp`; `__1_0`..`__7_0` are 0 bytes. They are
+retained deliberately -- `.metadata` enumerates every rank, and a DCP load into an
+8-rank mesh expects the files to be present. Do not prune them.

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/config.yaml

@@ -0,0 +1,466 @@
+checkpoint:
+  broadcast_via_filesystem: 'False'
+  dcp_allow_mismatched_size: 'False'
+  dcp_async_mode_enabled: 'False'
+  jit:
+    device: cuda
+    dtype: bfloat16
+    enabled: 'False'
+    input_shape: null
+    strict: 'True'
+  keys_not_to_resume: []
+  load_ema_to_reg: 'False'
+  load_from_object_store:
+    bucket: ''
+    credentials: ''
+    enabled: 'False'
+  load_path: ''
+  load_training_state: 'False'
+  only_load_scheduler_state: 'False'
+  override_optimizer_lr: 'False'
+  save_iter: '125'
+  save_to_object_store:
+    bucket: ''
+    credentials: ''
+    enabled: 'False'
+  strict_resume: 'False'
+  type:
+    _target_: <class 'rcm.checkpointers.dcp_distill.DistributedCheckpointer_Distill'>
+    callbacks: null
+    disable_async: 'False'
+  verbose: 'True'
+dataloader_train:
+  _target_: <function create_dataloader at 0x74c3f432e840>
+  batch_size: '1'
+  max_samples: '0'
+  num_workers: '1'
+  prefetch_factor: '2'
+  sample_offset: '6000'
+  shard_shuffle_buffer: '0'
+  shuffle_buffer: '0'
+  tar_path_pattern: /fsx/gstevenw/context_compression/datasets/sfdmd_c3-3_rollout_81920train_256val_seed0/parts/train/part-*/shard-00000.tar
+  uniform_shard_samples: '128'
+dataloader_val:
+  _target_: <class 'torch.utils.data.dataloader.DataLoader'>
+  batch_sampler: null
+  batch_size: '1'
+  collate_fn: null
+  dataset: <function <lambda> at 0x74c3f42c8f40>
+  drop_last: 'False'
+  generator: null
+  in_order: 'True'
+  multiprocessing_context: null
+  num_workers: '0'
+  persistent_workers: 'False'
+  pin_memory: 'False'
+  pin_memory_device: ''
+  prefetch_factor: null
+  sampler: null
+  shuffle: null
+  timeout: '0'
+  worker_init_fn: null
+defaults:
+- _self_
+- trainer: standard
+- data_train: dummy
+- data_val: dummy
+- optimizer: fusedadamw
+- scheduler: lambdalinear
+- callbacks: basic
+- checkpoint: local
+- ckpt_type: dcp
+- model: fsdp_t2v_distill_rcm
+- net: null
+- net_compressor: null
+- net_teacher: null
+- net_causal_teacher: null
+- net_bidirectional_teacher: null
+- net_fake_score: null
+- optimizer_fake_score: fusedadamw
+- conditioner: text_nodrop
+- ema: power
+- tokenizer: wan2pt1_tokenizer
+- experiment: null
+job:
+  group: future-sf-scaleup
+  name: context_compress_future_sf_c3-3_81920train_cr4_source_aligned_gbs8_lr2e-4_10000it_clip5_cosine
+  project: Memory_Compression
+  wandb_mode: online
+model:
+  _recursive_: 'False'
+  _target_: <class 'rcm.models.t2v_model_causal.T2VCausalModel'>
+  config:
+    backward_timesteps:
+    - '0.9375'
+    - '0.8333333333333334'
+    - '0.625'
+    bidirectional_guidance: 5.0
+    bidirectional_teacher_ckpt: ''
+    causal_guidance: 3.0
+    causal_teacher_ckpt: ''
+    checkpoint:
+      bucket: ''
+      credentials: ''
+      enabled: false
+    chunk_t: 3
+    compressor_ckpt: ''
+    conditioner:
+      _target_: <class 'rcm.conditioner.TextConditioner'>
+      text:
+        _target_: <class 'rcm.conditioner.TextAttr'>
+        dropout_rate: '0.0'
+        empty_string_embeddings_path: negative_prompt/empty_string_umt5.pt
+        input_key:
+        - t5_text_embeddings
+        use_empty_string: 'False'
+    context_compression_ae_weight: 1.0
+    context_compression_cacheaware_branch: future
+    context_compression_compressed_context_checkpoint_mode: reentrant
+    context_compression_compressor_checkpoint_mode: default
+    context_compression_disable_compressed_context_checkpoint: false
+    context_compression_history_t: 0
+    context_compression_history_t_max: 20
+    context_compression_history_t_min: 1
+    context_compression_memory_rope_mode: source_aligned
+    context_compression_pad_sampled_history_t: false
+    context_compression_prefix_t_values:
+    - '3'
+    - '6'
+    - '9'
+    - '12'
+    - '15'
+    - '18'
+    context_compression_rollout_denoise_steps: 4
+    context_compression_rollout_detach_history: true
+    context_compression_rollout_loss_reduction: mean
+    context_compression_rollout_steps: 0
+    context_compression_sample_history_t: false
+    context_compression_sample_target_start_t: false
+    context_compression_static_fixed_noise: false
+    context_compression_static_t: -1.0
+    context_compression_static_t_values: ''
+    context_compression_target_start_t_max: 0
+    context_compression_target_start_t_min: 0
+    context_compression_target_t: 3
+    context_from_last_step: false
+    context_from_last_step_start_chunk: 0
+    dcm_skipping_interval_steps: 1
+    dcm_total_steps: 48
+    df_timestep_strategy: random
+    ema:
+      enabled: false
+      iteration_shift: 0
+      rate: 0.1
+    fake_score_ckpt: ''
+    first_chunk_t: 3
+    fsdp_shard_size: 1
+    generation_dit_frozen: true
+    grad_clip: true
+    input_caption_key: prompts
+    input_data_key: videos
+    input_latent_key: latents
+    loss_scale: 100.0
+    loss_scale_dmd: 1.0
+    loss_scale_fake_score: 1.0
+    loss_weighting: none
+    max_simulation_steps_fake: 4
+    neg_embed_path: /fsx/gstevenw/context_compression/assets/checkpoints/umT5_wan_negative_emb.pt
+    net:
+      _target_: <class 'rcm.networks.wan2pt1.WanModel'>
+      cross_attn_norm: 'True'
+      dim: '1536'
+      eps: 1e-06
+      ffn_dim: '8960'
+      freq_dim: '256'
+      in_dim: '16'
+      model_type: t2v
+      num_heads: '12'
+      num_layers: '30'
+      out_dim: '16'
+      patch_size:
+      - '1'
+      - '2'
+      - '2'
+      qk_norm: 'True'
+      sac_config:
+        determinism_check: default
+        every_n_blocks: 1
+        mode: block_wise
+      text_dim: '4096'
+      text_len: '512'
+    net_bidirectional_teacher: null
+    net_causal_teacher: null
+    net_compressor:
+      _target_: <class 'rcm.networks.wan2pt1.WanMemoryCompressor'>
+      cross_attn_norm: 'True'
+      dim: '1536'
+      eps: 1e-06
+      ffn_dim: '8960'
+      freq_dim: '256'
+      in_dim: '16'
+      max_memory_tokens_per_frame: '1024'
+      memory_ratio: '0.25'
+      model_type: t2v
+      num_heads: '12'
+      num_layers: '30'
+      out_dim: '16'
+      patch_size:
+      - '1'
+      - '2'
+      - '2'
+      qk_norm: 'True'
+      sac_config:
+        determinism_check: default
+        every_n_blocks: 1
+        mode: block_wise
+      text_dim: '4096'
+      text_len: '512'
+    net_fake_score: null
+    optimizer_fake_score:
+      _target_: <function get_base_optimizer at 0x74c3f828b6a0>
+      betas:
+      - '0.9'
+      - '0.99'
+      capturable: 'True'
+      eps: 1e-08
+      lr: '0.0001'
+      master_weights: 'True'
+      model: null
+      optim_type: fusedadam
+      weight_decay: '0.1'
+    p_D:
+      _target_: <class 'rcm.utils.timestep_utils.UniformShift'>
+      shift: '5.0'
+    p_G:
+      _target_: <class 'rcm.utils.timestep_utils.UniformShift'>
+      p_mean: '-0.8'
+      p_std: '1.6'
+      shift: '5.0'
+    precision: bfloat16
+    rectified_flow_t_scaling_factor: 1000.0
+    replayed_training: false
+    resolution: 480p
+    sample_timestep_shift: 3.0
+    sf_backward_timestep_schedules: []
+    sf_simulation_steps_per_chunk: []
+    shift3_backward: false
+    sigma_data: 1.0
+    sigma_max: 1600.0
+    state_ch: 16
+    state_t: 21
+    student_ckpt: /fsx/gstevenw/context_compression/assets/checkpoints/Causal_rCM_Wan2.1_T2V_1.3B_480p_TF-dCM-init_SF-DMD_c3-3_step4.pt
+    student_update_freq: 5
+    tangent_warmup: 0
+    text_encoder_class: umT5
+    text_encoder_path: /fsx/gstevenw/context_compression/assets/checkpoints/models_t5_umt5-xxl-enc-bf16.pth
+    tokenizer:
+      _target_: <class 'rcm.tokenizers.wan2pt1.Wan2pt1VAEInterface'>
+      chunk_duration: '81'
+      vae_pth: /fsx/gstevenw/context_compression/assets/checkpoints/Wan2.1_VAE.pth
+    training_type: cc_cacheaware_self_forcing
+model_parallel:
+  _cpu_offloading_context: null
+  autocast_dtype: torch.float32
+  barrier_with_L1_time: true
+  batch_p2p_comm: true
+  batch_p2p_sync: true
+  bf16: false
+  context_parallel_size: 1
+  cpu_offloading: false
+  cpu_offloading_activations: true
+  cpu_offloading_double_buffering: false
+  cpu_offloading_num_layers: 0
+  cpu_offloading_retain_pinned_cpu_buffers: false
+  cpu_offloading_weights: false
+  cross_entropy_fusion_impl: native
+  cross_entropy_loss_fusion: false
+  deallocate_pipeline_outputs: false
+  defer_embedding_wgrad_compute: false
+  delay_wgrad_compute: false
+  deterministic_mode: false
+  enable_autocast: false
+  ep_overlap_early_attn_memory_release: false
+  expert_model_parallel_size: 1
+  expert_tensor_parallel_size: 1
+  finalize_model_grads_func: null
+  fp16: false
+  grad_scale_func: null
+  grad_sync_func: null
+  gradient_accumulation_fusion: false
+  hierarchical_context_parallel_sizes: null
+  hybrid_context_parallel: false
+  max_seqlen_per_dp_cp_rank: null
+  microbatch_group_size_per_vp_stage: 1
+  mtp_standalone: false
+  no_sync_func: null
+  num_microbatches_with_partial_activation_checkpoints: null
+  overlap_dispatch_backward_with_experts_wgrad: false
+  overlap_moe_expert_parallel_comm: false
+  overlap_p2p_comm: false
+  overlap_p2p_comm_warmup_flush: false
+  param_sync_func: null
+  params_dtype: torch.float32
+  perform_initialization: true
+  pipeline_dtype: null
+  pipeline_model_parallel_comm_backend: null
+  pipeline_model_parallel_size: 1
+  sequence_parallel: false
+  tensor_model_parallel_size: 1
+  timers: null
+  tp_comm_atomic_ag: false
+  tp_comm_atomic_rs: false
+  tp_comm_bootstrap_backend: nccl
+  tp_comm_bulk_dgrad: true
+  tp_comm_bulk_wgrad: true
+  tp_comm_overlap: false
+  tp_comm_overlap_ag: true
+  tp_comm_overlap_disable_fc1: false
+  tp_comm_overlap_disable_qkv: false
+  tp_comm_overlap_rs: true
+  tp_comm_overlap_rs_dgrad: false
+  tp_comm_split_ag: true
+  tp_comm_split_rs: true
+  use_cpu_initialization: false
+  use_ring_exchange_p2p: false
+  use_te_rng_tracker: false
+  variable_seq_lengths: false
+  virtual_pipeline_model_parallel_size: null
+  wgrad_deferral_limit: 0
+optimizer:
+  _target_: <function get_base_optimizer at 0x74c3f828b6a0>
+  betas:
+  - '0.9'
+  - '0.999'
+  capturable: 'True'
+  eps: 1e-08
+  lr: '0.0002'
+  master_weights: 'True'
+  model: null
+  optim_type: fusedadam
+  weight_decay: '0.01'
+scheduler:
+  _target_: rcm.utils.lr_scheduler.LambdaWarmUpCosineScheduler
+  cycle_lengths:
+  - '10000'
+  f_max:
+  - '1.0'
+  f_min:
+  - '0.25'
+  f_start:
+  - 1e-06
+  verbosity_interval: '0'
+  warm_up_steps:
+  - '100'
+trainer:
+  callbacks:
+    compile_tokenizer:
+      _target_: <class 'rcm.callbacks.compile_tokenizer.CompileTokenizer'>
+      compile_after_iterations: '0'
+      dynamic: 'False'
+      enabled: 'False'
+    dataloader_speed:
+      _target_: <class 'rcm.callbacks.dataloading_monitor.DetailedDataLoadingSpeedMonitor'>
+      every_n: '5'
+      save_s3: 'False'
+      step_size: '1'
+    device_monitor:
+      _target_: <class 'rcm.callbacks.device_monitor.DeviceMonitor'>
+      every_n: '5'
+      log_memory_detail: 'True'
+      save_s3: 'False'
+      step_size: '1'
+      upload_every_n_mul: '10'
+    every_n_sample_ema:
+      _target_: <class 'rcm.callbacks.every_n_draw_distill.EveryNDrawSample_Distill'>
+      every_n: '100000000'
+      is_ema: 'True'
+      is_image: 'False'
+      is_sample: 'True'
+      n_sample_to_save: '64'
+      num_samples: '2'
+      num_sampling_step: '50'
+      run_at_start: 'False'
+      sample_fix: 'False'
+      save_s3: 'False'
+      show_all_frames: 'False'
+      step_size: '1'
+    every_n_sample_reg:
+      _target_: <class 'rcm.callbacks.every_n_draw_distill.EveryNDrawSample_Distill'>
+      every_n: '100000000'
+      is_ema: 'False'
+      is_image: 'False'
+      is_sample: 'True'
+      n_sample_to_save: '64'
+      num_samples: '2'
+      num_sampling_step: '50'
+      run_at_start: 'False'
+      sample_fix: 'False'
+      save_s3: 'False'
+      show_all_frames: 'False'
+      step_size: '1'
+    grad_clip:
+      _target_: <class 'rcm.callbacks.grad_clip.GradClip'>
+      clip_norm: '5.0'
+      force_finite: 'True'
+    heart_beat:
+      _target_: <class 'rcm.callbacks.heart_beat.HeartBeat'>
+      every_n: '10'
+      save_s3: 'False'
+      step_size: '1'
+      update_interval_in_minute: '20'
+    iter_speed:
+      _target_: <class 'rcm.callbacks.iter_speed.IterSpeed'>
+      every_n: '5'
+      hit_thres: '5'
+      save_s3: 'False'
+      save_s3_every_log_n: '10'
+    low_prec:
+      _target_: <class 'imaginaire.callbacks.low_precision.LowPrecisionCallback'>
+      config: null
+      trainer: null
+      update_iter: '1'
+    manual_gc:
+      _target_: <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>
+      every_n: '5'
+      warm_up: '5'
+    wandb:
+      _target_: <class 'rcm.callbacks.wandb_log.WandbCallback'>
+      logging_iter_multipler: '1'
+      save_logging_iter_multipler: '10'
+      save_s3: 'False'
+    wandb_10x:
+      _target_: <class 'rcm.callbacks.wandb_log.WandbCallback'>
+      logging_iter_multipler: '10'
+      save_logging_iter_multipler: '1'
+      save_s3: 'False'
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp:
+    broadcast_buffers: 'True'
+    find_unused_parameters: 'False'
+    static_graph: 'True'
+  distributed_parallelism: ddp
+  grad_accum_iter: '1'
+  grad_scaler_args:
+    enabled: 'False'
+  logging_iter: '5'
+  max_iter: '10000'
+  max_val_iter: null
+  memory_format: torch.preserve_format
+  profiling:
+    enable_memory_snapshot: 'False'
+    enable_profiling: 'False'
+    first_n_rank: '8'
+    profile_freq: '1'
+    profile_memory: 'True'
+    record_shape: 'True'
+    with_modules: 'True'
+    with_stack: 'True'
+  run_validation: 'False'
+  seed: '0'
+  timeout_period: '999999999'
+  type:
+    _target_: <class 'rcm.trainers.trainer_distillation.ImaginaireTrainer_Distill'>
+    config: null
+  validation_iter: '100'

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/metadata.json

@@ -0,0 +1,37 @@
+{
+  "chunk_t": 3,
+  "clip_norm": 5.0,
+  "code_commit": "44f4c8b11f671e811f896539c5faa96db77a078d",
+  "contents": "model/inference state only; optimizer and scheduler state excluded",
+  "dataset_manifest": "/fsx/gstevenw/context_compression/datasets/sfdmd_c3-3_rollout_81920train_256val_seed0/manifest.json",
+  "dataset_manifest_sha256": "488feab621a47d5f5b77e1fb6547b24d64912cba66b7d6cababa53c1d4c8dd78",
+  "first_chunk_t": 3,
+  "global_batch_size": 8,
+  "grad_clip": true,
+  "lr_schedule": {
+    "cosine_min_factor": 0.25,
+    "cycle_length_steps": 10000,
+    "f_max": 1.0,
+    "f_start": 1e-06,
+    "final_lr": 5e-05,
+    "peak_lr": 0.0002,
+    "scheduler_target": "rcm.utils.lr_scheduler.LambdaWarmUpCosineScheduler",
+    "type": "cosine",
+    "warmup_steps": 100
+  },
+  "memory_rope_mode": "source_aligned",
+  "model_family": "c3-3",
+  "optimizer_lr": 0.0002,
+  "optimizer_step": 8000,
+  "repo_path": "c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000",
+  "sample_exposure": 64000,
+  "shard_count": 8,
+  "training_run_dir": "context_compress_future_sf_c3-3_81920train_cr4_source_aligned_gbs8_lr2e-4_10000it_clip5_cosine",
+  "validation_arm": "c3-source-cosine",
+  "validation_campaign": "c3_3_fixed_noise_validation_20260804",
+  "validation_metrics": null,
+  "variant": "cosine_2e-4_to5e-5_clip5",
+  "wandb_group": "future-sf-scaleup",
+  "wandb_project": "Memory_Compression",
+  "wandb_run_id": "jhnefd4q"
+}

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/model/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:344b3c5dab9a31cf13e71af7da657dff898421db179bec89758f4d773b308684
+size 481484

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/model/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e545aec2e7dd16e9ea2554276f2534b61fb449e4a6b5e64fd56502a3a33baa10
+size 5681533593

c3-3/cosine_2e-4_to5e-5_clip5/source_aligned/iter_000008000/sha256_manifest.json

@@ -0,0 +1,52 @@
+{
+  "files": {
+    "README.md": {
+      "bytes": 1027,
+      "sha256": "81caf7d73f81d3567f691d8f288cf8f26e9c69e6dea2389d81cbd798b91da49b"
+    },
+    "config.yaml": {
+      "bytes": 13397,
+      "sha256": "0b0fcfa29819347476a2ea0e19993fbd2370fb9ec64e7707e8023fa38ae8f821"
+    },
+    "metadata.json": {
+      "bytes": 1422,
+      "sha256": "0630c37d150a57b220d844ff7f1949b504d64e058512c60d9eda6ea8e79c9edb"
+    },
+    "model/.metadata": {
+      "bytes": 481484,
+      "sha256": "344b3c5dab9a31cf13e71af7da657dff898421db179bec89758f4d773b308684"
+    },
+    "model/__0_0.distcp": {
+      "bytes": 5681533593,
+      "sha256": "e545aec2e7dd16e9ea2554276f2534b61fb449e4a6b5e64fd56502a3a33baa10"
+    },
+    "model/__1_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__2_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__3_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__4_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__5_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__6_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "model/__7_0.distcp": {
+      "bytes": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    }
+  }
+}