backup: sq8/baseline-50k/vqvae/training-config.yaml

sq8/baseline-50k/vqvae/training-config.yaml

@@ -0,0 +1,105 @@
+# AUTO-GENERATED by scripts/ablation_sweep/make_nlevels_configs.py
+# Number-of-levels ablation. The ladder is the repo squared-16 sqrt-space
+# curve resampled to N points with monotonicity enforced in sqrt space, so
+# every level stays a perfect square and N=16 reproduces squared-16 exactly.
+# Level count touches FOUR fields: manual_levels, vqvae_vocab_size (both
+# copies) and model.vocab_size -- all must have exactly N entries.
+task: summarization
+experiment: ablation-vqvae-sq8
+experiment_dir: /workspace/ncm/runs/ablation-vqvae-sq8/vqvae
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 1000
+  checkpoint_interval: 5000
+  val_iters: 100
+  n_epochs: 3
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 64
+  eval_batch_size: 16
+  demo_batch_size: 4
+  stop_after_iters: 50000
+model:
+  n_layers: 6
+  context_length: 1024
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.2
+  pre_quant_groupnorm: 4
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_l1_cascade: true
+    learned_l1_cascade_base: 4
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 9
+      - 36
+      - 100
+      - 196
+      - 324
+      - 625
+      - 1024
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  project: ncp-release
+  enable: false
+  group: ablation-vqvae-sq8
+  resume: null
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0003
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 1
+torch_compile:
+  enable: true
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: data/full_owt/train_gpt2.bin
+  validate_source: data/full_owt/valid_gpt2.bin
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>