Upload ablation-ncp-sq8 ncp config

sq8/baseline-50k/ncp/training-config.yaml

@@ -0,0 +1,150 @@
+# AUTO-GENERATED by scripts/ablation_sweep/make_nlevels_configs.py
+# Number-of-levels ablation. The ladder is the repo squared-16 sqrt-space
+# curve resampled to N points with monotonicity enforced in sqrt space, so
+# every level stays a perfect square and N=16 reproduces squared-16 exactly.
+# Level count touches FOUR fields: manual_levels, vqvae_vocab_size (both
+# copies) and model.vocab_size -- all must have exactly N entries.
+task: next-concept
+experiment: ablation-ncp-sq8
+experiment_dir: /workspace/ncm/runs/ablation-ncp-sq8/ncp
+model:
+  n_layer: 16
+  n_head: 16
+  bias: true
+  dropout: 0.1
+  n_embd: 1024
+  context_length: 1024
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 100000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: false
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_mlp: false
+  shared_head_mlp_mult: 2.0
+  shared_head_mlp_residual: true
+  shared_head_mlp_blocks: 2
+  n_kv_heads: 4
+  mlp_ratio: 3.5
+  in_vocab_size: 50304
+  predict_level0: true
+  bos_token_id: 50256
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: gpt2
+    special_tokens:
+      eos_token: <|endoftext|>
+  vqvae:
+    n_layers: 6
+    vqvae_vocab_size:
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    embed_dim: 384
+    compression_factor: 1
+    in_vocab_size: 50304
+    context_length: 1024
+    checkpoint_path: ncm/runs/ablation-vqvae-sq8/vqvae/checkpoint-iter-50000.pt
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_l1_cascade: true
+      learned_l1_cascade_base: 4
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 9
+        - 36
+        - 100
+        - 196
+        - 324
+        - 625
+        - 1024
+      aux:
+        fine_drop:
+          prob: 0.5
+          min_keep: 1
+    dropout: 0.2
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: null
+dataset:
+  train_source: data/full_owt/train_gpt2.bin
+  validate_source: data/full_owt/valid_gpt2.bin
+wandb:
+  project: ncp-colm
+  group: ablation-ncp-sq8
+  enable: false
+  resume: null
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  level_loss_alpha: 0.25
+  lr: 0.0005
+  reset_lr: false
+  warmup_iters: 300
+  lr_decay_iters: null
+  min_lr: 1.0e-05
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.01
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 8
+  corruption:
+    mode: none
+training:
+  seed: 55
+  log_interval: 10
+  eval_interval: 1000
+  checkpoint_interval: 1000
+  val_iters: 100
+  total_iters: null
+  tqdm_interval: 10
+  n_epochs: 3
+  batch_size: 8
+  eval_batch_size: 16
+  demo_batch_size: 4
+  stop_after_iters: 5000
+  keep_last: 100
+torch_compile:
+  enable: true
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true