Archive stopped generator: latest 45k, best validation 44k, results and best Gen-PPL provenance

iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-44000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd84df67dc533dcd4e3554e83363275b247d59abfa8c06d0936343eea2277f31
+size 2565487307

iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-45000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:729d7432a0d99e7e1ccacd1526ffe0426e374df87c9037da52a0af0a60dee726
+size 2565487243

iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/EVAL_DONE.json

@@ -0,0 +1,5 @@
+{
+  "step": 25750,
+  "reference_dtype": "float32",
+  "sampling": "supplied-level0-untruncated"
+}

iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/summary.json

@@ -0,0 +1,35 @@
+{
+  "protocol": {
+    "sampling": "random",
+    "temperature": 1.0,
+    "top_k": 0,
+    "top_p": 1.0,
+    "n_provided_levels": 1,
+    "n_levels": 16,
+    "document_aware_validation_sampling": true
+  },
+  "reference_model": "gpt2-large",
+  "checkpoint": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-25750/ncp.pt",
+  "checkpoint_step": 25750,
+  "seeds": [
+    0,
+    1,
+    2,
+    3,
+    4
+  ],
+  "samples_per_seed": 128,
+  "total_scored_samples": 640,
+  "total_scored_tokens": 160504,
+  "per_seed_mean_ppl": {
+    "0": 205.27390254957734,
+    "1": 191.09995809495,
+    "2": 199.86937893942272,
+    "3": 185.8896512015558,
+    "4": 168.47801285382266
+  },
+  "mean_gen_ppl": 190.1221807278657,
+  "gen_ppl_se": 6.371510436577567,
+  "mean_of_seed_median_ppl": 179.17403564453124,
+  "mean_entropy_nats": 4.466862099671543
+}

iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp-config.yaml

@@ -0,0 +1,179 @@
+task: next-concept
+experiment: iclr-debug-two-stage-135b-20260916-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000
+model:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: gpt2
+    special_tokens:
+      eos_token: <|endoftext|>
+      pad_token: <|pad|>
+  vqvae:
+    n_layers: 6
+    context_length: 256
+    embed_dim: 384
+    in_vocab_size: 50304
+    vqvae_vocab_size:
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    compression_factor: 1
+    dropout: 0.1
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: 0.2
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 4
+        - 9
+        - 16
+        - 25
+        - 36
+        - 49
+        - 64
+        - 81
+        - 100
+        - 121
+        - 144
+        - 169
+        - 196
+        - 225
+        - 256
+      aux:
+        fine_drop:
+          prob: 0.5
+          min_keep: 1
+    checkpoint_path: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae.pt
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+wandb:
+  entity: mstok
+  project: owt-repro
+  group: owtsmall-repro-ctx256-sharedhead-ncp
+  enable: false
+  id: owtsmall-repro-ctx256-sharedhead-ncp-5ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0005
+  reset_lr: true
+  warmup_iters: 300
+  lr_decay_iters: null
+  min_lr: 1.0e-05
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 4
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+  level_loss_alpha: 1.0
+training:
+  seed: 55
+  log_interval: 10
+  eval_interval: 5000
+  checkpoint_interval: 2500
+  val_iters: 100
+  total_iters: null
+  tqdm_interval: 10
+  n_epochs: 5
+  batch_size: 96
+  eval_batch_size: 96
+  demo_batch_size: 4
+  keep_last: 8
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true

iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59e5405c493f4c5310dd9f9c53e3a5563de8902b387c281bb079dfa347a5c88d
+size 427469662

iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae-config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-two-stage-135b-20260916-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f85bbb9c51879b97f5dfa175815335643d2c18169568455e88731b84715fcdab
+size 1300864085

iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp-config.yaml

@@ -0,0 +1,179 @@
+task: next-concept
+experiment: iclr-debug-two-stage-135b-20260916-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000
+model:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: gpt2
+    special_tokens:
+      eos_token: <|endoftext|>
+      pad_token: <|pad|>
+  vqvae:
+    n_layers: 6
+    context_length: 256
+    embed_dim: 384
+    in_vocab_size: 50304
+    vqvae_vocab_size:
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    compression_factor: 1
+    dropout: 0.1
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: 0.2
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 4
+        - 9
+        - 16
+        - 25
+        - 36
+        - 49
+        - 64
+        - 81
+        - 100
+        - 121
+        - 144
+        - 169
+        - 196
+        - 225
+        - 256
+      aux:
+        fine_drop:
+          prob: 0.5
+          min_keep: 1
+    checkpoint_path: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae.pt
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+wandb:
+  entity: mstok
+  project: owt-repro
+  group: owtsmall-repro-ctx256-sharedhead-ncp
+  enable: false
+  id: owtsmall-repro-ctx256-sharedhead-ncp-5ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0005
+  reset_lr: true
+  warmup_iters: 300
+  lr_decay_iters: null
+  min_lr: 1.0e-05
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 4
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+  level_loss_alpha: 1.0
+training:
+  seed: 55
+  log_interval: 10
+  eval_interval: 5000
+  checkpoint_interval: 2500
+  val_iters: 100
+  total_iters: null
+  tqdm_interval: 10
+  n_epochs: 5
+  batch_size: 96
+  eval_batch_size: 96
+  demo_batch_size: 4
+  keep_last: 8
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true

iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0cf1f72beda0fc582af91014fd1dc7dcc8a2a23c7f38b91c853020da1db04a
+size 427469662

iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae-config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-two-stage-135b-20260916-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a16ecf6a1b8fb925e65bbcc120fce5fee7f0705e882c25351a1c4a6f6be0ceba
+size 1300864085

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/README.md

@@ -0,0 +1,7 @@
+# Stopped original generator run — September 17, 2026
+
+Stopped at user request to test a fresh generator with faster learning-rate decay. The original training schedule decayed over 135B positions. Latest retained resumable checkpoint: step 45,000 (47.186B positions). Best validation-loss checkpoint: step 44,000. Best measured generation-PPL checkpoint: step 12,875 (13.500B positions), 174.9645 ± 3.4274 seed SE under strict FP32 GPT-2 Large scoring, five seeds × 128 samples. This checkpoint and its full results were previously backed up and are verified again in this archive.
+
+Step 22,000 scored 189.300829 in FP32 and 189.300857 in FP64 on identical samples; it remains available with its checkpoint and precision audit. The scheduled step 25,750 evaluation scored 190.1222 ± 6.3715 but had TF32 enabled; it is not a strict FP32/FP64 comparison. No generation-PPL claim is made for steps 44,000 or 45,000.
+
+All generators here use the frozen final tokenizer from step 128,747. The restart uses that exact tokenizer again, fresh generator and optimizer state, the same initialization seeds, and a 27B-position cosine horizon followed by minimum LR to 135B. Samples remain level-0 conditioned, untruncated temperature-1, top-k 0, top-p 1. Full training states include optimizer, scheduler, token count, and per-rank RNG state. See the archive manifest for SHA-256 verification and validation-curve.json for all validation measurements.

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/STOPPED.json

@@ -0,0 +1,10 @@
+{
+  "stage": "stopped-by-user",
+  "updated_utc": "2026-09-17T22:25:15.271975+00:00",
+  "last_logged_step": 45320,
+  "latest_resumable_step": 45000,
+  "best_validation_step": 44000,
+  "best_measured_gen_ppl_step": 12875,
+  "best_measured_gen_ppl": 174.96447636439655,
+  "reason": "Fresh generator with same final tokenizer, cosine decay over first 27B positions then constant minimum LR to 135B"
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/STOP_REQUESTED.json

@@ -0,0 +1,21 @@
+{
+  "requested_utc": "2026-09-17T22:23:44.577915+00:00",
+  "reason": "User requested stopping and fresh generator with LR decay over 27B then minimum LR to 135B",
+  "retained_checkpoints": [
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-45000.pt",
+      "step": 45000,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/manual-checkpoints/checkpoint-iter-45000.pt"
+    },
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/best_checkpoint.pt",
+      "step": 44000,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/manual-checkpoints/checkpoint-iter-44000.pt"
+    },
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/milestone-iter-12875.pt",
+      "step": 12875,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/manual-checkpoints/checkpoint-iter-12875.pt"
+    }
+  ]
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/backup-manifest.json

@@ -0,0 +1,244 @@
+{
+  "repo": "iskhare/iclr-debug",
+  "run": "iclr-debug-two-stage-135b-20260916",
+  "stopped": {
+    "stage": "stopped-by-user",
+    "updated_utc": "2026-09-17T22:25:15.271975+00:00",
+    "last_logged_step": 45320,
+    "latest_resumable_step": 45000,
+    "best_validation_step": 44000,
+    "best_measured_gen_ppl_step": 12875,
+    "best_measured_gen_ppl": 174.96447636439655,
+    "reason": "Fresh generator with same final tokenizer, cosine decay over first 27B positions then constant minimum LR to 135B"
+  },
+  "files": [
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/manual-checkpoints/checkpoint-iter-44000.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-44000.pt",
+      "size_bytes": 2565487307,
+      "sha256": "dd84df67dc533dcd4e3554e83363275b247d59abfa8c06d0936343eea2277f31"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp-config.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp-config.yaml",
+      "size_bytes": 3528,
+      "sha256": "a5dccd1fecd77d5c6fd4be4c27f148ff4df9e0597a5b12bba60fff329a8f6d84"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/ncp.pt",
+      "size_bytes": 427469662,
+      "sha256": "59e5405c493f4c5310dd9f9c53e3a5563de8902b387c281bb079dfa347a5c88d"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae-config.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae-config.yaml",
+      "size_bytes": 2204,
+      "sha256": "42de51ae35c5eabaadf092d006402364b3d9abcf5c88e3729e5bb504347fe2a8"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-44000/vqvae.pt",
+      "size_bytes": 1300864085,
+      "sha256": "f85bbb9c51879b97f5dfa175815335643d2c18169568455e88731b84715fcdab"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/manual-checkpoints/checkpoint-iter-45000.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-45000.pt",
+      "size_bytes": 2565487243,
+      "sha256": "729d7432a0d99e7e1ccacd1526ffe0426e374df87c9037da52a0af0a60dee726"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp-config.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp-config.yaml",
+      "size_bytes": 3528,
+      "sha256": "15f635dc8642a4d907d7591ace6cbfbf8d4453aea3417cd849664d0be56ec0f8"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/ncp.pt",
+      "size_bytes": 427469662,
+      "sha256": "ed0cf1f72beda0fc582af91014fd1dc7dcc8a2a23c7f38b91c853020da1db04a"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae-config.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae-config.yaml",
+      "size_bytes": 2204,
+      "sha256": "01fe4a876d94ccbdd2db2e1eb1875c79a9ab8cd847e674bdb7cba973c03a4827"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/exports/step-45000/vqvae.pt",
+      "size_bytes": 1300864085,
+      "sha256": "a16ecf6a1b8fb925e65bbcc120fce5fee7f0705e882c25351a1c4a6f6be0ceba"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/EVAL_DONE.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/EVAL_DONE.json",
+      "size_bytes": 97,
+      "sha256": "8c09c167a7014823111474214e1dc7d55933435a3817c760ddb3f76585652652"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-0.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-0.json",
+      "size_bytes": 139915,
+      "sha256": "d55c6f8d7ac4939703de6dbe18a7ea18df01a78000f399affec0045bcec58135"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-0.samples.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-0.samples.json",
+      "size_bytes": 140297,
+      "sha256": "891672e4d85cf834dab622f001a4ea6e089f20e6e68074cdca91cae25cf4a517"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-1.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-1.json",
+      "size_bytes": 140716,
+      "sha256": "f09db410bace324d426e2fece48cc8d63c9fd5efa2d6a401de9b884e2f8ce21d"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-1.samples.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-1.samples.json",
+      "size_bytes": 141102,
+      "sha256": "70515a25f18fe739ad3ed422dad50dadaa6a3187beff59b60f436ce1753e0031"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-2.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-2.json",
+      "size_bytes": 142006,
+      "sha256": "8739454acd33efe78f2d588916571ef6cfe993341a8f48b5828b1dc1842ea28c"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-2.samples.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-2.samples.json",
+      "size_bytes": 142388,
+      "sha256": "89cf46368cd424c80e8e61ae8c1bca0950a54ee0a72cce66f54f5747cdbaa5fe"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-3.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-3.json",
+      "size_bytes": 137427,
+      "sha256": "0ceb6857343c0f15ff92fbe8fce64d72dcd7c2471c9969996438e53d0ced6ba9"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-3.samples.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-3.samples.json",
+      "size_bytes": 137813,
+      "sha256": "c0ddf888841b4e19fc3799c97cf481037cc44e87e42f76c4375483d284778cb3"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-4.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-4.json",
+      "size_bytes": 143033,
+      "sha256": "d70cbec97e155b63900639db7b1c150085699e55e7b7dfee54ae4e4a1c3a214e"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-4.samples.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/random-seed-4.samples.json",
+      "size_bytes": 143416,
+      "sha256": "abcb84609659b85030b8da9d3529e7c8aa54234e6b245bfd6d86d06103fdfc86"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/summary.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-25750/summary.json",
+      "size_bytes": 859,
+      "sha256": "24c26188704f1be2574c31246171bef7e0ced59f30bddbc991f9264ef0cf9a83"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/STOPPED.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/STOPPED.json",
+      "size_bytes": 397,
+      "sha256": "a9e18ead0604b4b8acbc2f04f17f9100840caf5324f829e894aa91828c48f3e9"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/STOP_REQUESTED.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/STOP_REQUESTED.json",
+      "size_bytes": 1047,
+      "sha256": "a91a6b80b5e16074974391e42eeff8c74c38b43912a6ca88cbfcbef959d372d5"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/manifest.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/manifest.json",
+      "size_bytes": 35157,
+      "sha256": "aaf11243953c582e83ba62448365d4039807f62205185e5f2228cda6348dd58c"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/git-provenance.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/git-provenance.json",
+      "size_bytes": 59,
+      "sha256": "96f601db3a051fd136086ca5f54c87ce2a72449e528d826a3b882541953e3fa2"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/config.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/config.yaml",
+      "size_bytes": 4572,
+      "sha256": "b72db37de5c634702e488e7f582115543c1ed551a01e1a4cef078a6d9dff2cc8"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/progress.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/progress.json",
+      "size_bytes": 402,
+      "sha256": "aa62ccc08980f01e258661f18f3918f58280eff4370febbd1a54d2d9256b3f57"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/hf-stop-backup-20260917/validation-curve.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/validation-curve.json",
+      "size_bytes": 11963,
+      "sha256": "d7dc5f8e605ecebb236ff081e2cf86af46e22c6f83304cf4e6450852a7c283ae"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/training-to-12875.log",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-12875.log",
+      "size_bytes": 507886,
+      "sha256": "d7e0f5288e8cecf8d031255bc9afa7606ba49fd4cb2b2347d1f53753f1cbd572"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/training-to-25750.log",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-25750.log",
+      "size_bytes": 803561,
+      "sha256": "79b2b12a7b102153c5bf4102e4d1e89259c491e4b128abddcd4a24d5ec6d8351"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/training-to-4769.log",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-4769.log",
+      "size_bytes": 242468,
+      "sha256": "f48339ccc1150a063d5450a8c841b481a45c4719c1dcbf14aad03e785aac122b"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/training-to-51499.log",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-51499.log",
+      "size_bytes": 1217730,
+      "sha256": "23fc7b806f53de781d2f9a673229461c05f412f38aee8016f87459dedb6c0c93"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/training-to-954.log",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-954.log",
+      "size_bytes": 67236,
+      "sha256": "b2fd347d43dc10e8257954e96162cb24602072da8067cfbf80cc2b3a0ec5f3dd"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/milestone-iter-12875.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/milestone-iter-12875.pt",
+      "size_bytes": 2565487435,
+      "sha256": "fefa9b7bf94073fe1f4145462c8fd497bf2f6a5dd9bfe611bada5b1b1658942f"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-12875-fp32/summary.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-12875-fp32/summary.json",
+      "size_bytes": 969,
+      "sha256": "f677d9e6853198e694f03243866c2e0ef6956fcff661b1d6fe61e5f57e285040"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/evaluation/step-22000-fp32-fp64/comparison.json",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/evaluation/step-22000-fp32-fp64/comparison.json",
+      "size_bytes": 2475,
+      "sha256": "84381543ba0ce4468ef888334d228487752fc6a0b36da631a3bc00d69c561012"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/hf-stop-backup-20260917/README.md",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/README.md",
+      "size_bytes": 1447,
+      "sha256": "2b790471d8f85682208c1654e09237a57f7daeb398b6300ed7b3c1836c619f29"
+    }
+  ]
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/config.yaml

@@ -0,0 +1,223 @@
+task: mstok-next-concept
+experiment: iclr-debug-two-stage-135b-20260916-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>
+codec:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer_config:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+generator:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+objectives:
+  codec_weight: 1.0
+  ncp_weight: 1.0
+  soft_assignment_temperature: 1.0
+  prediction_temperature: 1.0
+  mstok_weight: 0.25
+  residual_weight: 1.0
+  reconstruction_weight: 1.0
+  teacher_ema_decay: 0.999
+  teacher_ema_warmup_steps: 1000
+  mstok_warmup_steps: 500
+optimization:
+  codec_lr: 0.001
+  codec_min_lr: 0.0001
+  codec_warmup_iters: 0
+  codec_lr_decay_iters: 128747
+  generator_lr: 0.0005
+  generator_min_lr: 1.0e-05
+  generator_warmup_iters: 225
+  generator_lr_decay_iters: 128747
+  beta_1: 0.9
+  codec_beta_2: 0.99
+  generator_beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  level_loss_alpha: 1.0
+  grad_accumulation_steps: 16
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+training:
+  log_interval: 10
+  eval_interval: 1000
+  checkpoint_interval: 1000
+  eval_batch_size: 4
+  val_iters: 25
+  keep_last: 3
+  seed: 55
+  codec_initialization_seed: 42
+  generator_initialization_seed: 55
+  total_iters: 128747
+  batch_size: 32
+  expected_world_size: 8
+  expected_global_batch_size: 4096
+  resume_checkpoint: null
+  milestone_steps:
+  - 954
+  - 4769
+  - 12875
+  - 25750
+  - 51499
+  - 77248
+  - 102997
+  - 128747
+  generation_eval_steps:
+  - 12875
+  - 25750
+  - 51499
+  - 77248
+  - 102997
+  - 128747
+torch_compile:
+  enable: true
+  scope: loss
+  mode: default
+  dynamic: false
+  fullgraph: true
+  backend: inductor
+mixed_precision:
+  enable: true
+wandb:
+  entity: mstok
+  project: iclr-debug
+  group: full-owt-ctx256-16sq-135b-v1
+  enable: true
+  id: iclr-debug-two-stage-135b-20260916-generator
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+export:
+  vqvae_config_template: /home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/vqvae.yaml
+  ncp_config_template: /home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/ncp-sharedhead.yaml
+iclr_debug:
+  stage: generator
+  target_positions: 135000000000
+  lr_horizon_positions: 135000000000
+  stop_step: 128747
+  pilot: false
+  tokenizer_checkpoint: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt
+  tokenizer_steps: 128747
+  budget_unit: input positions including padding; log non-padding tokens separately

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/git-provenance.json

@@ -0,0 +1,3 @@
+{
+  "commit": "1b5226d4bbdd34653cb5fdd58d436355fb76ac8c"
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/manifest.json

@@ -0,0 +1,756 @@
+{
+  "format": "iclr-debug-production-v1",
+  "variant": "two-stage",
+  "configs": {
+    "tokenizer": {
+      "task": "mstok-next-concept",
+      "experiment": "iclr-debug-two-stage-135b-20260916-tokenizer",
+      "experiment_dir": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer",
+      "dataset": {
+        "train_source": "/home/ubuntu/data/full_owt/train_gpt2.bin",
+        "validate_source": "/home/ubuntu/data/full_owt/valid_gpt2.bin",
+        "pad_token_id": 50257
+      },
+      "pre_tokenizer": {
+        "name": "hf",
+        "tokenizer": "hf",
+        "model_id": "gpt2",
+        "special_tokens": {
+          "eos_token": "<|endoftext|>",
+          "pad_token": "<|pad|>"
+        }
+      },
+      "codec": {
+        "n_layers": 6,
+        "context_length": 256,
+        "embed_dim": 384,
+        "in_vocab_size": 50304,
+        "vqvae_vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "compression_factor": 1,
+        "dropout": 0.1,
+        "pre_quant_groupnorm": 4,
+        "pre_quant_dropout": 0.2,
+        "vector_quantizer_config": {
+          "clss": "multiscale_residual_vector_quantizer",
+          "decay": 0.99,
+          "epsilon": 1e-05,
+          "commitment_cost": 0.25,
+          "learned_l1_sampling": true,
+          "learned_all_sampling": false,
+          "quant_resi": {
+            "enabled": true,
+            "ratio": 0.5,
+            "share_mode": 0,
+            "learnable_ratio": false
+          },
+          "levels": {
+            "use_manual_levels": true,
+            "manual_levels": [
+              1,
+              4,
+              9,
+              16,
+              25,
+              36,
+              49,
+              64,
+              81,
+              100,
+              121,
+              144,
+              169,
+              196,
+              225,
+              256
+            ]
+          },
+          "aux": {
+            "fine_drop": {
+              "prob": 0.5,
+              "min_keep": 1
+            }
+          }
+        }
+      },
+      "generator": {
+        "n_layer": 12,
+        "n_head": 12,
+        "bias": true,
+        "dropout": 0.1,
+        "n_embd": 768,
+        "context_length": 256,
+        "vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "attn_pattern": "block_diagonal",
+        "use_positional_encoding": true,
+        "use_level_encoding": true,
+        "prefix_len": 0,
+        "use_rope": true,
+        "rope_base": 10000,
+        "use_qk_norm": true,
+        "post_upsample_conv": {
+          "enabled": true,
+          "kernel_size": 3
+        },
+        "use_relu2": true,
+        "use_flex_attention": true,
+        "shared_output_head": true,
+        "shared_head_per_level_bias": false,
+        "shared_head_per_level_scale": false,
+        "shared_head_adapter_rank": 0
+      },
+      "objectives": {
+        "codec_weight": 1.0,
+        "ncp_weight": 1.0,
+        "soft_assignment_temperature": 1.0,
+        "prediction_temperature": 1.0,
+        "mstok_weight": 0.25,
+        "residual_weight": 1.0,
+        "reconstruction_weight": 1.0,
+        "teacher_ema_decay": 0.999,
+        "teacher_ema_warmup_steps": 1000,
+        "mstok_warmup_steps": 500
+      },
+      "optimization": {
+        "codec_lr": 0.001,
+        "codec_min_lr": 0.0001,
+        "codec_warmup_iters": 0,
+        "codec_lr_decay_iters": 128747,
+        "generator_lr": 0.0005,
+        "generator_min_lr": 1e-05,
+        "generator_warmup_iters": 225,
+        "generator_lr_decay_iters": 128747,
+        "beta_1": 0.9,
+        "codec_beta_2": 0.99,
+        "generator_beta_2": 0.99,
+        "weight_decay": 0.1,
+        "max_grad_norm": 1.0,
+        "level_loss_alpha": 1.0,
+        "grad_accumulation_steps": 1,
+        "corruption": {
+          "mode": "per_level",
+          "per_level_probs": [
+            0.85,
+            0.8321428571,
+            0.8142857143,
+            0.7964285714,
+            0.7785714286,
+            0.7607142857,
+            0.7428571429,
+            0.725,
+            0.7071428571,
+            0.6892857143,
+            0.6714285714,
+            0.6535714286,
+            0.6357142857,
+            0.6178571429,
+            0.6
+          ],
+          "skip_level0": true
+        }
+      },
+      "training": {
+        "log_interval": 10,
+        "eval_interval": 1000,
+        "checkpoint_interval": 1000,
+        "eval_batch_size": 4,
+        "val_iters": 25,
+        "keep_last": 3,
+        "seed": 55,
+        "codec_initialization_seed": 42,
+        "generator_initialization_seed": 55,
+        "total_iters": 128747,
+        "batch_size": 512,
+        "expected_world_size": 8,
+        "expected_global_batch_size": 4096,
+        "resume_checkpoint": null,
+        "milestone_steps": [
+          954,
+          4769,
+          12875,
+          25750,
+          51499,
+          77248,
+          102997,
+          128747
+        ],
+        "generation_eval_steps": []
+      },
+      "torch_compile": {
+        "enable": true,
+        "scope": "loss",
+        "mode": "default",
+        "dynamic": false,
+        "fullgraph": true,
+        "backend": "inductor"
+      },
+      "mixed_precision": {
+        "enable": true
+      },
+      "wandb": {
+        "entity": "mstok",
+        "project": "iclr-debug",
+        "group": "full-owt-ctx256-16sq-135b-v1",
+        "enable": true,
+        "id": "iclr-debug-two-stage-135b-20260916-tokenizer",
+        "resume": "allow",
+        "gradients_and_params": {
+          "enable": false,
+          "log": "all",
+          "log_freq": 1000
+        }
+      },
+      "export": {
+        "vqvae_config_template": "/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/vqvae.yaml",
+        "ncp_config_template": "/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/ncp-sharedhead.yaml"
+      },
+      "iclr_debug": {
+        "stage": "tokenizer",
+        "target_positions": 135000000000,
+        "lr_horizon_positions": 135000000000,
+        "stop_step": 128747,
+        "pilot": false,
+        "tokenizer_checkpoint": null,
+        "tokenizer_steps": null,
+        "budget_unit": "input positions including padding; log non-padding tokens separately"
+      }
+    },
+    "generator": {
+      "task": "mstok-next-concept",
+      "experiment": "iclr-debug-two-stage-135b-20260916-generator",
+      "experiment_dir": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator",
+      "dataset": {
+        "train_source": "/home/ubuntu/data/full_owt/train_gpt2.bin",
+        "validate_source": "/home/ubuntu/data/full_owt/valid_gpt2.bin",
+        "pad_token_id": 50257
+      },
+      "pre_tokenizer": {
+        "name": "hf",
+        "tokenizer": "hf",
+        "model_id": "gpt2",
+        "special_tokens": {
+          "eos_token": "<|endoftext|>",
+          "pad_token": "<|pad|>"
+        }
+      },
+      "codec": {
+        "n_layers": 6,
+        "context_length": 256,
+        "embed_dim": 384,
+        "in_vocab_size": 50304,
+        "vqvae_vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "compression_factor": 1,
+        "dropout": 0.1,
+        "pre_quant_groupnorm": 4,
+        "pre_quant_dropout": 0.2,
+        "vector_quantizer_config": {
+          "clss": "multiscale_residual_vector_quantizer",
+          "decay": 0.99,
+          "epsilon": 1e-05,
+          "commitment_cost": 0.25,
+          "learned_l1_sampling": true,
+          "learned_all_sampling": false,
+          "quant_resi": {
+            "enabled": true,
+            "ratio": 0.5,
+            "share_mode": 0,
+            "learnable_ratio": false
+          },
+          "levels": {
+            "use_manual_levels": true,
+            "manual_levels": [
+              1,
+              4,
+              9,
+              16,
+              25,
+              36,
+              49,
+              64,
+              81,
+              100,
+              121,
+              144,
+              169,
+              196,
+              225,
+              256
+            ]
+          },
+          "aux": {
+            "fine_drop": {
+              "prob": 0.5,
+              "min_keep": 1
+            }
+          }
+        }
+      },
+      "generator": {
+        "n_layer": 12,
+        "n_head": 12,
+        "bias": true,
+        "dropout": 0.1,
+        "n_embd": 768,
+        "context_length": 256,
+        "vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "attn_pattern": "block_diagonal",
+        "use_positional_encoding": true,
+        "use_level_encoding": true,
+        "prefix_len": 0,
+        "use_rope": true,
+        "rope_base": 10000,
+        "use_qk_norm": true,
+        "post_upsample_conv": {
+          "enabled": true,
+          "kernel_size": 3
+        },
+        "use_relu2": true,
+        "use_flex_attention": true,
+        "shared_output_head": true,
+        "shared_head_per_level_bias": false,
+        "shared_head_per_level_scale": false,
+        "shared_head_adapter_rank": 0
+      },
+      "objectives": {
+        "codec_weight": 1.0,
+        "ncp_weight": 1.0,
+        "soft_assignment_temperature": 1.0,
+        "prediction_temperature": 1.0,
+        "mstok_weight": 0.25,
+        "residual_weight": 1.0,
+        "reconstruction_weight": 1.0,
+        "teacher_ema_decay": 0.999,
+        "teacher_ema_warmup_steps": 1000,
+        "mstok_warmup_steps": 500
+      },
+      "optimization": {
+        "codec_lr": 0.001,
+        "codec_min_lr": 0.0001,
+        "codec_warmup_iters": 0,
+        "codec_lr_decay_iters": 128747,
+        "generator_lr": 0.0005,
+        "generator_min_lr": 1e-05,
+        "generator_warmup_iters": 225,
+        "generator_lr_decay_iters": 128747,
+        "beta_1": 0.9,
+        "codec_beta_2": 0.99,
+        "generator_beta_2": 0.99,
+        "weight_decay": 0.1,
+        "max_grad_norm": 1.0,
+        "level_loss_alpha": 1.0,
+        "grad_accumulation_steps": 16,
+        "corruption": {
+          "mode": "per_level",
+          "per_level_probs": [
+            0.85,
+            0.8321428571,
+            0.8142857143,
+            0.7964285714,
+            0.7785714286,
+            0.7607142857,
+            0.7428571429,
+            0.725,
+            0.7071428571,
+            0.6892857143,
+            0.6714285714,
+            0.6535714286,
+            0.6357142857,
+            0.6178571429,
+            0.6
+          ],
+          "skip_level0": true
+        }
+      },
+      "training": {
+        "log_interval": 10,
+        "eval_interval": 1000,
+        "checkpoint_interval": 1000,
+        "eval_batch_size": 4,
+        "val_iters": 25,
+        "keep_last": 3,
+        "seed": 55,
+        "codec_initialization_seed": 42,
+        "generator_initialization_seed": 55,
+        "total_iters": 128747,
+        "batch_size": 32,
+        "expected_world_size": 8,
+        "expected_global_batch_size": 4096,
+        "resume_checkpoint": null,
+        "milestone_steps": [
+          954,
+          4769,
+          12875,
+          25750,
+          51499,
+          77248,
+          102997,
+          128747
+        ],
+        "generation_eval_steps": [
+          12875,
+          25750,
+          51499,
+          77248,
+          102997,
+          128747
+        ]
+      },
+      "torch_compile": {
+        "enable": true,
+        "scope": "loss",
+        "mode": "default",
+        "dynamic": false,
+        "fullgraph": true,
+        "backend": "inductor"
+      },
+      "mixed_precision": {
+        "enable": true
+      },
+      "wandb": {
+        "entity": "mstok",
+        "project": "iclr-debug",
+        "group": "full-owt-ctx256-16sq-135b-v1",
+        "enable": true,
+        "id": "iclr-debug-two-stage-135b-20260916-generator",
+        "resume": "allow",
+        "gradients_and_params": {
+          "enable": false,
+          "log": "all",
+          "log_freq": 1000
+        }
+      },
+      "export": {
+        "vqvae_config_template": "/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/vqvae.yaml",
+        "ncp_config_template": "/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/config/repro-ctx256/ncp-sharedhead.yaml"
+      },
+      "iclr_debug": {
+        "stage": "generator",
+        "target_positions": 135000000000,
+        "lr_horizon_positions": 135000000000,
+        "stop_step": 128747,
+        "pilot": false,
+        "tokenizer_checkpoint": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt",
+        "tokenizer_steps": 128747,
+        "budget_unit": "input positions including padding; log non-padding tokens separately"
+      }
+    }
+  },
+  "sources": {
+    "config/repro-ctx256/alignment-generator.yaml": "f62f36d078a3a373a19ea716a3ebc7eda5ae2fb7d670c5b935c734826bb76895",
+    "config/repro-ctx256/alignment-joint.yaml": "bb511950d20335e4237e406464ab26760a7fd9c9c9472c0507addd88d33d5178",
+    "config/repro-ctx256/alignment-tokenizer.yaml": "2d32955296f7d15990f284328c1cf8f52071abdba28033a65e272bb49cf5dd56",
+    "config/repro-ctx256/alignment.yaml": "3a486fa2ed28eca4fc406232669399b0625d274376ac43271f4ccf187b1e9077",
+    "config/repro-ctx256/joint.yaml": "8b4889111687a52e2d7d71995bda45a7f01f2777674c88191f338dca4c1732d2",
+    "config/repro-ctx256/mstok-semantic-eostok-matched10ep.yaml": "ae8a72c0c9f1cb5d2e301d83a1cdc08985a1f895bf627a5f75b1b3ce627f5e58",
+    "config/repro-ctx256/mstok-semantic-eostok.yaml": "2c9c48b21325c0e479990701b28116d78f611101724846747d173e52145fe15a",
+    "config/repro-ctx256/mstok-semantic-gear-matched10ep.yaml": "164cb8ac39fa407bb337f559ad8a9c62a34708fb900b8829d3ccf182961f2d0c",
+    "config/repro-ctx256/mstok-semantic-gear.yaml": "8cb8c998551daef0bf7fd9ec77b13b34c06662028d8820767e1432b5a9e8be54",
+    "config/repro-ctx256/mstok-semantic.yaml": "e0b00dbfa3eb838103f657c19bde20e36375c7fbe692623452ea093003f1ac98",
+    "config/repro-ctx256/mstok-w1-pilot.yaml": "21ee22eeccaaea2563c92ccc84f6df3871ea5da6a5ce980656088ea577dc1438",
+    "config/repro-ctx256/mstok-w1.yaml": "b6bf6079114b3c997fab89e85a53318bb2ec5dae55678513599d84152eab701b",
+    "config/repro-ctx256/mstok.yaml": "d9bb216f4eeb58857d72be399d5f8847d197ce70d0e504879f0788045483dc87",
+    "config/repro-ctx256/ncp-sharedhead.yaml": "86c9efbcb2b4f74b7253aa3bc4393de4a6a05e20ea30da573d1ca094ec600645",
+    "config/repro-ctx256/ncp.yaml": "93663654c4ce5ad0931db052edf507fdcb7c0ff9fc9722f4018820deee5c5acf",
+    "config/repro-ctx256/substitution-generator.yaml": "064f55887b9b2b8f3b0b52c094d4a193510dc1148ad7a5c7f53e317971f87c7c",
+    "config/repro-ctx256/substitution-joint.yaml": "b1576f4ecfe6fc6029a0cd868271b80a1d988b7f3667c6b1a8d24a6a4015df3b",
+    "config/repro-ctx256/substitution-tokenizer.yaml": "e29be5046f0fafbfe4c06973be4cd036a004e845348911cc0e6e05a7fca5d425",
+    "config/repro-ctx256/substitution.yaml": "ebba1c7e664de0e84913bdfd38e794837781c42c1b48cde90135ff9417856a95",
+    "config/repro-ctx256/vqvae.yaml": "05fde46e57c3e6b411e8a6afcac251f178e4b8a0856a0b99332bb284fb4710ce",
+    "data/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/serialized_dataset.py": "5622c2d94236cd1b9b4ca18e8d5c272a86b54d193d99048fb9b7cd2687e8dbc6",
+    "data/tinysentences/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinysentences/clean.py": "918a9a91aae484a5e06d6d212c2c0974946a4032e061c62a7b65851735bfc8dd",
+    "data/tinysentences/clean_parsed.py": "009a087f967e171bc6098484dbfd58635302900f6b07629629898f1d00fa70a3",
+    "data/tinysentences/generate_meta.py": "662714f1370e05567d412add168a7c3a05f4b1c9adb047fad374900bc1936a7e",
+    "data/tinysentences/generate_metadata_char.py": "afd3336ea03ff0653ccaf41c121d9b93debf7a380db6cd881b62b308ee6ee68c",
+    "data/tinysentences/generate_training_data_word.py": "82e2edcb7ea4db0609ccf2b047a0b207e7b8fcff4fc0b4ceee521ff278e8a8e8",
+    "data/tinysentences/parse_summaries.py": "2317159364dbbe7206f252d2939cab5d375395e0a536be0a1f60c5df88b31702",
+    "data/tinysentences/prepare.py": "35263b287f93337fda4207ed1991e9e8a820ab75139411bb7dd5f0a8a25bcbd9",
+    "data/tinysentences/sentences_only.py": "5e4d72bf612df27dad46cf606a40402606d897043ae24274dc2dfa88ec0b3e79",
+    "data/tinysentences/stats.py": "2665e9e6fac4a8f6c3e5bd8b07cbfd941ba6359a43981d89bdef58c136fdd3a9",
+    "data/tinysentences/summarize/merge_summaries_char.py": "bce94f4d9a824c2d25e45448023d59b7fbc42aba22b33efbc067cdbba93e1934",
+    "data/tinysentences/summarize/merge_summaries_word.py": "f6079110332cc38b2dd3571c30bedcca97098c39498f45eec1ef6e60cce7ac81",
+    "data/tinysentences/summarize/summarize_gemini.py": "282a438efce0dee351fd9833a59c56e4480bed4314dae40844201794ada3bd96",
+    "data/tinysentences/summarize/summarize_gpt.py": "fc08cbc37d6161e5b0057a1ede43272d96f7274524917d6c4d42c22f505c6e09",
+    "data/tinysentences/summarize/summarize_llama.py": "7f3f8230652a5b88ea2856b273d497cd5b633580b430c909e4f996cd0232832d",
+    "data/tinysentences/summarize/summarize_qwen.py": "b07e3c226ab0fdf5b21b025263adfa7c2625d640f0fac3c787bc4a2c9e32107f",
+    "data/tinysentences/summarize/summarize_together.py": "8c26ea052f2a64c2653d39f1ea5c13983dfbc9547f21cd7d7ea9d9b39033bffe",
+    "data/tinysentences/test.py": "9749244da77c078f550a78751664b45bb92067682f3aa27c2338419f1186c844",
+    "data/tinysentences/test_regex.py": "46652e189a0d5f5c0c7a2c90b857082004bf6727ef1d3ad42b631ac7bb0aa88f",
+    "data/tinysentences/tinysentences_char_dataset.py": "152ab9c30aec669a9db14df9e90412597870105eff9cfb737b876aad2323a206",
+    "data/tinysentences/tinysentences_dataset.py": "f90251d21871f6a7ad5ebd2ad0a6e59cd0b02ca245f7e46f93e6d970bf657bc7",
+    "data/tinysentences/tinysentences_vae.py": "30f7adde5413348faf5066448706747fe26a7fedd510ffea720aa0bf8283f6f8",
+    "data/tinysentences/to_csv.py": "c06c74648ed95b4494835c631c94fe1a00a9ff47a272f5e38546a7f94ff9be8b",
+    "data/tinysentences/tokenize/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinysentences/tokenize/common.py": "e877c74df2a48d15991492b126cd0c5fe26731d61f3aa36d83ebf2cb9bda34a0",
+    "data/tinysentences/tokenize/tokenizer.py": "c16abc50b92a163f6978542b36b03c9a14aa00153c73117c4d52084d5de25c2a",
+    "data/tinysentences/tokenize/train_bpe.py": "fa8e55978dd5ca90117200648b468fb22822bf6981f15f0394c815e72eae1bdb",
+    "data/tinystories/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinystories/download.py": "5afce37e912f6965b6722f0bc028076a655fea78304f1a213e4862dbe223cdcb",
+    "data/tinystories/stats.py": "216650fe19b78e8431c4512ff87a5c84af64d9e56ab8fdaf3952ca035725ea1b",
+    "data/wikitext/parse_wikitext.py": "ab583e0328f82afc3ba242410e8076be42ef1a16a51d66f3eb46a713cca591d0",
+    "evaluation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "evaluation/eval_random_gen_ppl_checkpoint.py": "221ea1917dc2d3bc19be39da0e0b8cca49295d0400a05da276cd50956876da8d",
+    "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+    "evaluation/gen_ppl_ncp.py": "96d68adee372e5186d0470659171eb03829e3d37f611e8f5917b45fb61e3a4ad",
+    "evaluation/gen_ppl_ntp.py": "b2638622bc041fa88e9d538be4229b3723ebec19c5b99fd6e5b73db4776dc952",
+    "evaluation/guess.py": "c27803225dd21b2deff1e7e494b9e3666180b93776340164ce6e953dacc50656",
+    "evaluation/ppl.py": "f3b581cb4849b0b2aa518729b8338838c7055697c3a26028dedb819c335ac838",
+    "evaluation/run_genppl_multiseed.py": "f8c9ac982f4caa4335b5f4d1a22d70f56dc11a706106a3dfb31d344eb8c93c2e",
+    "evaluation/run_genppl_release_ts.py": "d1bc5c00089118d9cf6c05c77459329ad40c2f8bc347b5aeedda8f5318de8796",
+    "evaluation/run_genppl_sweep.py": "ade4f39e373125848fdd32938e8d7bf86a0ec5108cca9bd75f0a966d092a798e",
+    "evaluation/show_8k_samples.py": "30ad825d5a24375e178750dfaaac45bc486bc3587f2e7d6d305d53bf37f8d226",
+    "evaluation/show_low_sample.py": "d696f1e54ca67e87565dce5864e0e6e2ad4194108cfb1c9c1e06eebd97008da6",
+    "evaluation/summarize_gen_ppl.py": "307f3d0d85884879295d5268aaf7f399f654b1b330aaa5728d6824ad24c6d15a",
+    "evaluation/sweep_gen_ppl_checkpoints.py": "3a9141e27d84b49c5493194dfc5a70bfa0a915cdf5ceb7a46afcf27cc6f33acb",
+    "evaluation/sweep_gen_ppl_ncp.py": "d91c2959cfed51d224ccb1c0d0f2c3b9a35c4027cc9ab8c65aa7603d587de761",
+    "evaluation/sweep_gen_ppl_ncp_scaling.py": "05249c1e00c5eb05042b6ea5b1ca008ecb8f0e4a77537ee755a27c153c25a17c",
+    "evaluation/sweep_gen_ppl_ncp_yolo.py": "4b26ef25b27af2f7fcbcb5ff3c312d47d2a4746ba759155d28802e978a681755",
+    "evaluation/sweep_gen_ppl_ntp.py": "14a04074741ee44580ec6557870d42534dd172d26c5be3e943ed5baf1b727303",
+    "evaluation/sweep_random_llama_yolo.py": "4062b54289fcd9407d15feb1ffd241b8722fd717c700350c22472fecec5ae46c",
+    "evaluation/val_oracle_ppl.py": "1a23f3c2897b6a6a3d239ed533979de35352d303bae094f4006bef4156f935c0",
+    "evaluation/val_vqvae_ppl.py": "4c293c502d3faf982d4ad66566b7a037faa7d28191d929de1621cb67f5ae8b5f",
+    "models/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "models/masked_lm.py": "80279614d74efc7b4c25306789d74e2f71045c54e4e4f485ee365f664705d7f6",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/next_token.py": "d0d292b9f5c032a351cbca1a8f0a6d41c3062890cb3e3f5b67d4e23a7cffeb77",
+    "models/quant.py": "b7b73abd4e55e10454cb680dea82124c38c222d37236c4c76b5c7d21123ac1b1",
+    "models/semantic_input.py": "b06658d4bc641397cbb973255eadfd2d6087fde244d801d3c70d29e3ce841bf2",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "scripts/benchmark_iclr_debug.py": "dba86e8d578eec2188c633a479ccf673d58f3e5580b9074869cae1a352b8a043",
+    "scripts/benchmark_mstok_compile.py": "aad909da6f05ba25ac3d8f6b3cf450103bed55d0b93b1490764c0ef54f35b606",
+    "scripts/eval_owt_ckpts_gpt2_large.py": "4977a2d614698f60032fb6fd7fef614eb503d5b3883f499146a4fc048214a509",
+    "scripts/evaluate_mstok_sidecar.py": "9ba33e865fa0fad34bd2ab375d7fe38f37c441e2d96348f0e389e29dd9643455",
+    "scripts/evaluations/sweep_gen_ppl_shared_vocab_ncp.py": "2311ac71f9beba282426faa21390a5898c7d183a95f529755d6e1d6e8fc24050",
+    "scripts/neurips/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "scripts/neurips/eval_gen_ppl_ncp_owt.py": "b92eff1ce198eb4ac8c88da7113b41d095c12acad92db541685175993db3535c",
+    "scripts/neurips/sweep_gen_ppl_owt_gpt2.py": "02adb1629cfa40ff51a75152e4da9c9ee00ead42b7ecda9e78cf7101e6826a67",
+    "scripts/preflight_owtsmall_mstok_ctx256.py": "a4f80921435b6cd136d089ec39bfa5442359d0dffbeefaaa1019178b8364657a",
+    "scripts/prepare_full_owt.py": "b1cb6b74397625c2b0643c4ee744ca3c89dfa6bbb7eeb7e9681e3a36f91a2c24",
+    "scripts/r1_lab/diagnose_at_40k.py": "b1582dcc29ccebc9255eaa77a7d56a7dbdef589ec5d5a9b0af150dede98874ab",
+    "scripts/r1_lab/inspect_summaries.py": "4154e8a6c1b3f68df5c623a57a1a7632b594c46c5f6d1d85cb08c476d6aa9cea",
+    "scripts/r1_lab/overnight_pipeline.py": "53f4e03ce1a5fb43fbca3ae455288f45ae8350b0a16f3d5318763011de6a8f96",
+    "scripts/r1_lab/prime_rope_qknorm_score_cache.py": "a791cdec03b415bfccdfd8d89c7e659aff8a2a5696d16ab7b4350ff608d0391b",
+    "scripts/r1_lab/sweep_gen_ppl_r1lab.py": "c7c0acf9aa6f07bb0a985cf0d5e5692c696e658ebb0470b7c92d9332565f9c06",
+    "scripts/r1_lab/sweep_gen_ppl_r1lab_v15.py": "524aa8680aa1b9fea2689509848c19e5a74a3ad72e78b0388bbca7acd43be22e",
+    "scripts/r1_lab/sweep_gen_ppl_rope_qknorm.py": "7822df057355b0a3a71ea5145742107c6e30f906859096237ff61d765fe36d50",
+    "scripts/r1_lab/train_variant.py": "9d0a2091784609ac9728226afd82f56fd4c1ac2dc08e89e1b17b60d34bb3d55e",
+    "scripts/run_alignment.py": "6312cd9d0976899763589fdcbd258c7681f6e9e96d7396cf35ff022d4695b862",
+    "scripts/run_iclr_debug.py": "a48a14067af036a0eba46d8de69922c615c05f6bbbab11587dca292ea57682ce",
+    "scripts/run_mstok_semantic.py": "8a7a38a50dd6694950bfa0e7b79718ec653c7e70be99fe13cbf2ee05c28f46a0",
+    "scripts/run_mstok_w1_pilot.py": "35224e462d0d11eb3e4326b0daa8bad7c99e1e30cdc6383310f8e2c8b5bb708f",
+    "scripts/run_substitution.py": "2cd22eaa58dab8b5231984eee340a3a2bdd0b88cc83333f7133447471e7f73f1",
+    "scripts/smoke_test_masked_l0.py": "d24f1270b3219a31212b68bcbb786f649c7c72f96e53f602d87ad6c17f3f73e5",
+    "scripts/test0_diagnostics.py": "ed389c666beef015ccac004f314a5a1428def303b46ac0e442f0236337344549",
+    "scripts/test0_r1_similarity.py": "4a486878bebd418a3b5454ddbf02585726ccbacc2700fdf591aa524766f9c9c9",
+    "scripts/test0_r1_uniqueness.py": "3ee59d4397d5b268151dbb8ca779c7f9bf8461d129d97fcd1a517c48ec2a69a1",
+    "scripts/test0_residual_norms.py": "3c4f68e5960e84796dc0bca4fd4583bf3b598552654f9b565fd5fcabd420f9e1",
+    "scripts/test_gen_equiv.py": "ae0e9c513422ee58bdc050ed6218ee54fc555dfb15ea8968c9ffc62a7a7ea99a",
+    "scripts/training_budget.py": "bb94c19c6dfbdd542946b10476b8d6958b8a189abd141cf62288fdb5fa9a2d38",
+    "scripts/vqvae_utilization.py": "5f90327f1507a5042b2f77b119fe4fc0df4e2131e5bbc21069dffd1fadc059c5",
+    "scripts/xsum/codec_ceiling.py": "ddb751e9152d5c88966e588d89cdceee4c5d91c4139aaedf369f65a45ecf2276",
+    "scripts/xsum/eval_merge.py": "6e313d6ed25227d0418b7422d58e09ceda4fc6dfae7d0fd663417b306241135c",
+    "scripts/xsum/eval_q0.py": "82fe04dccce28d7fc9c6ec8b4c9c91404a49527f9437913f44f51a6853ba9abc",
+    "scripts/xsum/eval_shard.py": "07192f6ba6219da1e8123ceeb99748c86da85322859ed5fa25cebcc3816f7ea8",
+    "scripts/xsum/eval_shard_cfg.py": "95ac4a8e2e3a73a21fa4303c62df80c086f23bcfe5ab34eb07ae90b585052ecb",
+    "scripts/xsum/eval_test.py": "7f92d578c31893e788e5fa7d9eef3098ba5602b3642873ce2ce031342c8be868",
+    "scripts/xsum/eval_uncond_rep.py": "ba4740b29bc091d866e2dfd91881e9a7d5c46a1bfa993d13eb63c0467397ec28",
+    "scripts/xsum/make_warmstart.py": "78793d6ef11fbf658c2837ef12cb843fc4817add8ba73c822b5e408901de3318",
+    "scripts/xsum/mbr_fast.py": "3db6e21d2dcce2e3bcf6f22474535d7eba0232b071b4271d2b172f787aed5c85",
+    "scripts/xsum/mbr_select.py": "107a061014c62395b7f5fd986fba9e55dd738054e2474022005cb7e9e0c9b8fb",
+    "scripts/xsum/mbr_shard_cfg.py": "3e7162375028c391276149eff288e9de40c874a3108ac53de0248198f4d08791",
+    "scripts/xsum/pool_append.py": "e50252f48c4d8d905be0de3eb56ca1d720f8f8545c6fdcb58ed27f0360c4e2ad",
+    "scripts/xsum/prep_xsum_le48.py": "3688ed1abc49d18f75d34f8382ff68a5afbd753b36cefdc0652d0bffb6a5f347",
+    "scripts/xsum/prep_xsum_mix30.py": "4bd47e52637c4b6bca5fd4956c1787b506a7f51b8a5bb0b68f19f728c4d48afb",
+    "scripts/xsum/prep_xsum_mix_split.py": "ec2fe74f08068ae8fda4ed59a86537a1d3f48276d205099bbfe16f8428ca36d5",
+    "scripts/xsum/prep_xsum_pass_uncond.py": "82ea50248d811e5182e63d0b4dfd6f995dd2f47eb963b5bb1c5d09bc533db7c1",
+    "scripts/xsum/prep_xsum_test_all.py": "7389fbdf5612ba524801c5b04109f810b139152ca50101b60acbb9b1e767b256",
+    "scripts/xsum/score_checkpoints.py": "41b412c06cd92cbd4caa6b586da7ddc7e913607a47c1a3aac0ee0a36a20a36be",
+    "tokenization/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "tokenization/bpe_tokenizer.py": "0c8af84868a6aa40f981768f8ab49d28e501d55cce444edb1ad315de53c3445a",
+    "tokenization/build_word_vocab.py": "778f9666f4e7c642dc471773ee81c92c26975db37de1ebfffb605cae211351b5",
+    "tokenization/byte_tokenizer.py": "eb567a1772ae69738884ed9b94330aa39387c49e54d76e4b9752e67674fad810",
+    "tokenization/compression_ratio.py": "ed07ab1cc38d6176362631ce655bc974dd3623b87693f51279d5047e6bc6890f",
+    "tokenization/hf_tokenizer.py": "2ca8fae097f9c640bcb52b4788c37de8e0d9bb2fa971e30a5abb39f90ff97d4c",
+    "tokenization/tokenize.py": "f8ab54338613b6bd84513fdda85be173fd32274c35d97ef39a5c17d584f8108f",
+    "tokenization/tokenize_doc_level.py": "a06c81cf088fb81d2317ed1c3a1417a0e48ae0ea04962824f13f5d3beddc8902",
+    "tokenization/tokenize_vqvae.py": "14730eec3da87f6528e5fbd1166d5da79e10ac1700f7cd0be73bc40716bdebfc",
+    "tokenization/tokenizer.py": "e527d47e4ed8634c673d48a872a07c9cc8828d30988331de2d4bd3b7c21ddcb7",
+    "tokenization/train_bpe.py": "bf4e2a29eeba874024ed95f32fe720ae2320f72cb978361692db73e71be98683",
+    "tokenization/vqvae_tokenizer.py": "d7fa9d595865aa3e0b0d185bfd04721eb297fe5f657c4fca4ce69fc96aca308f",
+    "tokenization/word_tokenizer.py": "a405621bf35ed6cbb29abadd9d1d651e0783e6801fc7645b35e799d5de381ac2",
+    "train_mstok.py": "077b7c99c12daad2bbd986918d0f487411779de05a472af71124098da399b79d",
+    "trainer/alignment_trainer.py": "54f8ee7cca4d407223f2ce5146e6fc561ad748e20f2c702beb12ae040cd290c4",
+    "trainer/corruption.py": "f0f24c94652e6e9cf8e4ef30ae1a302e3232d64e765c931696d454db96e8b57d",
+    "trainer/joint_trainer.py": "8c8ebb55d21b6e223be1f7595c51f1b9257ccac40f8cff28260c1088656e2e49",
+    "trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
+    "trainer/ncp_trainer.py": "73fc23f78f403dc39800b924dfb18ad42ffeab847fed30e931183c3d3acaee68",
+    "trainer/ntp_trainer.py": "5076d7aa29fac70869e1e309a15af6c4d3a91d6de8e00ac113a586841a9866e1",
+    "trainer/onpolicy.py": "6ff1fe4103d71001afe2d73e574d20c7242381e420929db14020141464ec6821",
+    "trainer/plain_two_stage.py": "4c5973ed87100b1b478aa1df0bb16cd3d1eb5844dffe5894003dd67c463f6c9a",
+    "trainer/scheduled_sampling.py": "f53515e23c2cec1231265f21a341ff77d11fa8a6129e0e382d540cc85571f5ed",
+    "trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
+    "trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
+    "trainer/substitution_trainer.py": "33c7833b2e8dfabbb9ee56b258708055d3194b8d7453a1b8d93f59b44ff91465",
+    "utils/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "utils/arg_utils.py": "e9eb7971f6273e9630b56fd87a03725b29183a8c80abb014c4c84f8b1bdb6445",
+    "utils/benchmark.py": "ccf436ef72a4e626c7f671d8e63a683057bf502a3852e5a7142410d1dc3e5501",
+    "utils/data.py": "78cce11e777813755cd171ccaa3247f89ab99ac4079e0157e55c72ce8a5a7fe5",
+    "utils/dist.py": "b2849d3ce03a8b1737e3dd61216e76e4a9e82d63ad1ea74cddf72e2285eaca64",
+    "utils/experiment_config.py": "616a70fefaac99ab0d856a1b6c36787a28fb9af8a4c55fc6d4765012fbaaeb38",
+    "utils/iclr_training.py": "10c6857fa2319952fe0df18067730b42e3ff755244e2b059f6c95882f8d27c8e",
+    "utils/logging.py": "901e67515cc65a35f575f05d16a3b9ef4a5b2cb0ea252cd7abbc80ecce9e88e8",
+    "utils/lr.py": "4d1ea2c25340afc4bdde8b582dab36f174e0b2965fa7bafa2a7539019dbe2b7e",
+    "utils/misc.py": "be4a323116a0956a77960d1baf613f268057d6a36023663d2af9f8e4728cbe77",
+    "utils/plot_levels.py": "916b39885ced191960c26c334a3b16f9c99751f6c29c46bd69d14fee414965f3",
+    "utils/registry.py": "f7a44c6d18433100873d5df462b9f08a8bdeb11cf3d92226334c7e5ece318318",
+    "utils/train.py": "43eef988000f3a5e20856653d725337a0fe0dbe276f0cf80e1b200292abe812d",
+    "train_iclr_debug.py": "e0a0adf4a30ef55925f9017de24d434299749332e58a507a6424265f7a5f796e",
+    "config/iclr-debug/README.md": "1c757d2ff37026d08c22324ec44cfa760b0599af62c8ad9b728ada471a8f2905",
+    "config/iclr-debug/benchmark-16sq.yaml": "18c9cef7032f9d3806d8b0c8b20fe0fc7d1ea98ad1d2440e24a67ee873c97f9b",
+    "config/iclr-debug/full-owt-audit.json": "fefa394834f6e9f65944d3b514633c29061071ebe6ca60fce21a6edef9748c1e",
+    "config/iclr-debug/training-defaults.yaml": "3b144b8d53f96fe035e36edca96aac456e6cf21202373a467a3cd875fe86b474",
+    "config/iclr-debug/NCM-BUDGET-AUDIT.md": "e734caeab7edb612de7ff508a8c0ec52445530ab7a35895b5b793d6a7b752e4b",
+    "config/iclr-debug/TRAINING.md": "91555457c5cf6af078075673ce96cb440bc8e22aeae5211c252d94f92ea87ac7",
+    "config/iclr-debug/BENCHMARK.md": "2781ea506212e12a0640a4b0d2a2c881355ec85756ca4014bccb1b1aedccdf59",
+    "config/iclr-debug/HF-MODEL-CARD.md": "de1fe176ebde262eacdf32af7d3f8c6497a764dd98e5a9094dad26de4aade38e"
+  },
+  "dataset": {
+    "dataset": "hazyresearch/ncm-tokenized-datasets",
+    "revision": "951e04517fd53d57521d8e09abd6511a662787d0",
+    "dtype": "little-endian uint16",
+    "eos_id": 50256,
+    "offset_units": "token positions; [0, each EOS position + 1] with final sentinel",
+    "sampling": "length-weighted within-segment windows; pad short segments with 50257",
+    "caveat": "EOS segments are not independently verified source-document identities",
+    "splits": {
+      "train": {
+        "bytes": 18070847356,
+        "tokens": 9035423678,
+        "sha256": "b24b78a07f56974fd0c9855dc16f25b836d2cb172be23eb5d3c650cd6d73f56a",
+        "min_token_id": 0,
+        "max_token_id": 50256,
+        "segments": 8009770,
+        "trailing_tokens": 0,
+        "eos_only_segments": 0,
+        "length_percentiles": {
+          "min": 132.0,
+          "p25": 416.0,
+          "p50": 718.0,
+          "p75": 1249.0,
+          "p99": 7610.0,
+          "max": 131288.0
+        },
+        "segments_shorter_than_256": 673011,
+        "expected_padding_fraction_at_256": 1.5955357567300495e-05,
+        "offsets_status": "created",
+        "offsets_sha256": "7e84564946b8cf8190dbdc1e86bdf2fe8cfd9235a517328c80b4df038ea0028b",
+        "offset_entries": 8009771
+      },
+      "valid": {
+        "bytes": 9186834,
+        "tokens": 4593417,
+        "sha256": "1f2ee17e08327a39d4e2417948d02c7b998d626d2dcb2661197a6e208f58266f",
+        "min_token_id": 0,
+        "max_token_id": 50256,
+        "segments": 3999,
+        "trailing_tokens": 0,
+        "eos_only_segments": 0,
+        "length_percentiles": {
+          "min": 143.0,
+          "p25": 433.0,
+          "p50": 734.0,
+          "p75": 1292.5,
+          "p99": 7122.719999999997,
+          "max": 24868.0
+        },
+        "segments_shorter_than_256": 363,
+        "expected_padding_fraction_at_256": 1.6064002171981108e-05,
+        "offsets_status": "created",
+        "offsets_sha256": "09a748ce1dcec7de36c77c48e39ccbf5c0293cc073689fa7c5f48bdfb9bb9445",
+        "offset_entries": 4000
+      }
+    },
+    "overlap": {
+      "exact_train_segment_matches": 0,
+      "unique_validation_segments_in_train": 0,
+      "unique_validation_segments": 3999,
+      "near_duplicates_checked": false
+    }
+  },
+  "hf_repo": null,
+  "generation_evaluation": true,
+  "versions": {
+    "torch": "2.13.0+cu130",
+    "transformers": "4.44.2",
+    "numpy": "1.26.4",
+    "hydra-core": "1.3.6"
+  }
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/progress.json

@@ -0,0 +1,14 @@
+{
+  "step": 45000,
+  "checkpoint": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/generator/checkpoint-iter-45000.pt",
+  "nonpadding_tokens": 47185162118,
+  "validation": {
+    "loss": 5.5970091938972475,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.5970091938972475,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.62951505016722
+  }
+}

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/training-to-954.log

@@ -0,0 +1,185 @@
+wandb: Currently logged in as: iskhare (mstok). Use `wandb login --relogin` to force relogin
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+wandb: Tracking run with wandb version 0.17.9
+wandb: Run data is saved locally in /home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/wandb/run-20260916_151853-iclr-debug-two-stage-135b-20260916-generator
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run iclr-debug-two-stage-135b-20260916-generator
+wandb: ⭐️ View project at https://wandb.ai/mstok/iclr-debug
+wandb: 🚀 View run at https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-two-stage-135b-20260916-generator
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-two-stage-20260916/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+{"step": 1, "stop_at": 954, "target": 128747, "training/loss": 9.871083557605743, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.871083557605743, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.004180602006688963, "training/generator_lr": 4.444444444444444e-06, "training/grad_norm": 0.14711065590381622, "training/positions": 1048576, "training/nonpadding_tokens": 1048576, "throughput/steps_per_second": 0.04057870272065834, "throughput/raw_tokens_per_second": 42549.85378401704, "throughput/peak_reserved_gib": 23.4921875}
+{"step": 10, "stop_at": 954, "target": 128747, "training/loss": 9.844660883148512, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.844660883148512, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.006423320791527313, "training/generator_lr": 2.4444444444444445e-05, "training/grad_norm": 0.11732996255159378, "training/positions": 10485760, "training/nonpadding_tokens": 10485517, "throughput/steps_per_second": 0.4079115964308802, "throughput/raw_tokens_per_second": 427715.296526003, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 20, "stop_at": 954, "target": 128747, "training/loss": 9.761483243852854, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.761483243852854, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.10068392035953178, "training/generator_lr": 4.666666666666667e-05, "training/grad_norm": 0.07661070674657822, "training/positions": 20971520, "training/nonpadding_tokens": 20971118, "throughput/steps_per_second": 0.40710573381980913, "throughput/raw_tokens_per_second": 426874.82896467246, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 30, "stop_at": 954, "target": 128747, "training/loss": 9.691915126144886, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.691915126144886, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.12911201400501673, "training/generator_lr": 6.88888888888889e-05, "training/grad_norm": 0.06473804265260696, "training/positions": 31457280, "training/nonpadding_tokens": 31456759, "throughput/steps_per_second": 0.4075730020312962, "throughput/raw_tokens_per_second": 427366.4180592442, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 40, "stop_at": 954, "target": 128747, "training/loss": 9.61763765886426, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.61763765886426, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.14390415969899664, "training/generator_lr": 9.111111111111112e-05, "training/grad_norm": 0.05417146906256676, "training/positions": 41943040, "training/nonpadding_tokens": 41942305, "throughput/steps_per_second": 0.4073528609142874, "throughput/raw_tokens_per_second": 427131.71613483626, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 50, "stop_at": 954, "target": 128747, "training/loss": 9.540263690799474, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.540263690799474, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.15787488242056857, "training/generator_lr": 0.00011333333333333333, "training/grad_norm": 0.046574149280786514, "training/positions": 52428800, "training/nonpadding_tokens": 52427940, "throughput/steps_per_second": 0.40740229747828155, "throughput/raw_tokens_per_second": 427187.17895186803, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 60, "stop_at": 954, "target": 128747, "training/loss": 9.472409730404616, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.472409730404616, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.2101503710284281, "training/generator_lr": 0.00013555555555555556, "training/grad_norm": 0.04066415876150131, "training/positions": 62914560, "training/nonpadding_tokens": 62913625, "throughput/steps_per_second": 0.40773109145099445, "throughput/raw_tokens_per_second": 427533.97896613204, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 70, "stop_at": 954, "target": 128747, "training/loss": 9.417974783480167, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.417974783480167, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.279105808423913, "training/generator_lr": 0.00015777777777777776, "training/grad_norm": 0.047600157558918, "training/positions": 73400320, "training/nonpadding_tokens": 73399129, "throughput/steps_per_second": 0.4075350686449746, "throughput/raw_tokens_per_second": 427321.0592417155, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 80, "stop_at": 954, "target": 128747, "training/loss": 9.377109003067016, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.377109003067016, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.29349459134615385, "training/generator_lr": 0.00017999999999999998, "training/grad_norm": 0.03833989426493645, "training/positions": 83886080, "training/nonpadding_tokens": 83884677, "throughput/steps_per_second": 0.4075239805912369, "throughput/raw_tokens_per_second": 427311.2259640483, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 90, "stop_at": 954, "target": 128747, "training/loss": 9.345919600129127, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.345919600129127, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.42629402696488294, "training/generator_lr": 0.00020222222222222223, "training/grad_norm": 0.05508098751306534, "training/positions": 94371840, "training/nonpadding_tokens": 94370271, "throughput/steps_per_second": 0.4076887400406052, "throughput/raw_tokens_per_second": 427485.86064373294, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 100, "stop_at": 954, "target": 128747, "training/loss": 9.32499888986349, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.32499888986349, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.560940439485786, "training/generator_lr": 0.00022444444444444446, "training/grad_norm": 0.053070779889822006, "training/positions": 104857600, "training/nonpadding_tokens": 104855825, "throughput/steps_per_second": 0.4077148678501129, "throughput/raw_tokens_per_second": 427511.6263445223, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 110, "stop_at": 954, "target": 128747, "training/loss": 9.301467409729957, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.301467409729957, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.7047286528010034, "training/generator_lr": 0.0002466666666666667, "training/grad_norm": 0.06223422288894653, "training/positions": 115343360, "training/nonpadding_tokens": 115341579, "throughput/steps_per_second": 0.40769303410734853, "throughput/raw_tokens_per_second": 427496.8863163266, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 120, "stop_at": 954, "target": 128747, "training/loss": 9.274017015099526, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.274017015099526, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.8985648777173914, "training/generator_lr": 0.00026888888888888893, "training/grad_norm": 0.13982895016670227, "training/positions": 125829120, "training/nonpadding_tokens": 125827327, "throughput/steps_per_second": 0.407567096817899, "throughput/raw_tokens_per_second": 427364.58703240904, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 130, "stop_at": 954, "target": 128747, "training/loss": 9.245860026031732, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.245860026031732, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.0634781955476589, "training/generator_lr": 0.00029111111111111113, "training/grad_norm": 0.13594666123390198, "training/positions": 136314880, "training/nonpadding_tokens": 136312889, "throughput/steps_per_second": 0.4075819043854277, "throughput/raw_tokens_per_second": 427372.53285114735, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 140, "stop_at": 954, "target": 128747, "training/loss": 9.211429408937693, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.211429408937693, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.2399420594690636, "training/generator_lr": 0.0003133333333333334, "training/grad_norm": 0.12958887219429016, "training/positions": 146800640, "training/nonpadding_tokens": 146798407, "throughput/steps_per_second": 0.4074633294391767, "throughput/raw_tokens_per_second": 427246.4075174417, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 150, "stop_at": 954, "target": 128747, "training/loss": 9.174244021624327, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.174244021624327, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.4370149848453178, "training/generator_lr": 0.0003355555555555556, "training/grad_norm": 0.15638326108455658, "training/positions": 157286400, "training/nonpadding_tokens": 157284143, "throughput/steps_per_second": 0.40757154239395027, "throughput/raw_tokens_per_second": 427368.759465577, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 160, "stop_at": 954, "target": 128747, "training/loss": 9.141117287427187, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.141117287427187, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.598191889632107, "training/generator_lr": 0.00035777777777777777, "training/grad_norm": 0.07898727059364319, "training/positions": 167772160, "training/nonpadding_tokens": 167769767, "throughput/steps_per_second": 0.40762492446874615, "throughput/raw_tokens_per_second": 427420.1691007672, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 170, "stop_at": 954, "target": 128747, "training/loss": 9.106735324859619, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.106735324859619, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.7872367527173914, "training/generator_lr": 0.00038, "training/grad_norm": 0.152617409825325, "training/positions": 178257920, "training/nonpadding_tokens": 178255314, "throughput/steps_per_second": 0.4078569739411609, "throughput/raw_tokens_per_second": 427660.3469537818, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 180, "stop_at": 954, "target": 128747, "training/loss": 9.07505406215787, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.07505406215787, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.9343106579222409, "training/generator_lr": 0.0004022222222222222, "training/grad_norm": 0.12514373660087585, "training/positions": 188743680, "training/nonpadding_tokens": 188740985, "throughput/steps_per_second": 0.4078358538169566, "throughput/raw_tokens_per_second": 427643.25851287006, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 190, "stop_at": 954, "target": 128747, "training/loss": 9.04273925423622, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.04273925423622, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.1132240933319397, "training/generator_lr": 0.00042444444444444447, "training/grad_norm": 0.08854150027036667, "training/positions": 199229440, "training/nonpadding_tokens": 199226665, "throughput/steps_per_second": 0.40812692470204626, "throughput/raw_tokens_per_second": 427948.8331809752, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 200, "stop_at": 954, "target": 128747, "training/loss": 9.015308622270823, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.015308622270823, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.225790068457358, "training/generator_lr": 0.00044666666666666666, "training/grad_norm": 0.12497750669717789, "training/positions": 209715200, "training/nonpadding_tokens": 209712037, "throughput/steps_per_second": 0.40810996751045797, "throughput/raw_tokens_per_second": 427918.4826255066, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 210, "stop_at": 954, "target": 128747, "training/loss": 8.979814836382866, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.979814836382866, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.3991348113503346, "training/generator_lr": 0.0004688888888888889, "training/grad_norm": 0.1000954806804657, "training/positions": 220200960, "training/nonpadding_tokens": 220197684, "throughput/steps_per_second": 0.40812199630957774, "throughput/raw_tokens_per_second": 427942.3186237535, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 220, "stop_at": 954, "target": 128747, "training/loss": 8.958572793006898, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.958572793006898, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.4984322742474916, "training/generator_lr": 0.0004911111111111111, "training/grad_norm": 0.1701861172914505, "training/positions": 230686720, "training/nonpadding_tokens": 230683308, "throughput/steps_per_second": 0.4082000847540737, "throughput/raw_tokens_per_second": 428023.260549935, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 230, "stop_at": 954, "target": 128747, "training/loss": 8.925016617774963, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.925016617774963, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.66686317673495, "training/generator_lr": 0.0004999999981701273, "training/grad_norm": 0.08466679602861404, "training/positions": 241172480, "training/nonpadding_tokens": 241168894, "throughput/steps_per_second": 0.4083869075949109, "throughput/raw_tokens_per_second": 428217.60408604913, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 240, "stop_at": 954, "target": 128747, "training/loss": 8.900285533070564, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.900285533070564, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.77888044784699, "training/generator_lr": 0.0004999999835311453, "training/grad_norm": 0.15479880571365356, "training/positions": 251658240, "training/nonpadding_tokens": 251654582, "throughput/steps_per_second": 0.40832028543840454, "throughput/raw_tokens_per_second": 428151.9117178053, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 250, "stop_at": 954, "target": 128747, "training/loss": 8.87479942664504, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.87479942664504, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.900166897470736, "training/generator_lr": 0.0004999999542531824, "training/grad_norm": 0.0811324343085289, "training/positions": 262144000, "training/nonpadding_tokens": 262140041, "throughput/steps_per_second": 0.4084259037338934, "throughput/raw_tokens_per_second": 428253.30681396864, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 260, "stop_at": 954, "target": 128747, "training/loss": 8.84714948758483, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.84714948758483, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.0518590614548495, "training/generator_lr": 0.0004999999103362402, "training/grad_norm": 0.13342252373695374, "training/positions": 272629760, "training/nonpadding_tokens": 272625566, "throughput/steps_per_second": 0.4083502604907626, "throughput/raw_tokens_per_second": 428176.68651324033, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 270, "stop_at": 954, "target": 128747, "training/loss": 8.82384713292122, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.82384713292122, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.1786456155936453, "training/generator_lr": 0.0004999998517803213, "training/grad_norm": 0.15548212826251984, "training/positions": 283115520, "training/nonpadding_tokens": 283111224, "throughput/steps_per_second": 0.4085605127784436, "throughput/raw_tokens_per_second": 428402.5809299389, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 280, "stop_at": 954, "target": 128747, "training/loss": 8.802917934954166, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.802917934954166, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.2786763168896322, "training/generator_lr": 0.0004999997785854293, "training/grad_norm": 0.09455807507038116, "training/positions": 293601280, "training/nonpadding_tokens": 293596820, "throughput/steps_per_second": 0.40845573756965664, "throughput/raw_tokens_per_second": 428290.18480374414, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 290, "stop_at": 954, "target": 128747, "training/loss": 8.785858216881753, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.785858216881753, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.3791038487667224, "training/generator_lr": 0.0004999996907515685, "training/grad_norm": 0.180866539478302, "training/positions": 304087040, "training/nonpadding_tokens": 304082455, "throughput/steps_per_second": 0.40844399653324864, "throughput/raw_tokens_per_second": 428279.46655889106, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 300, "stop_at": 954, "target": 128747, "training/loss": 8.776521760970354, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.776521760970354, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.4013377926421406, "training/generator_lr": 0.0004999995882787442, "training/grad_norm": 0.0964689552783966, "training/positions": 314572800, "training/nonpadding_tokens": 314568065, "throughput/steps_per_second": 0.40854529070440465, "throughput/raw_tokens_per_second": 428384.6585663012, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 310, "stop_at": 954, "target": 128747, "training/loss": 8.75076204687357, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.75076204687357, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.5759628448996654, "training/generator_lr": 0.0004999994711669624, "training/grad_norm": 0.11090482026338577, "training/positions": 325058560, "training/nonpadding_tokens": 325053596, "throughput/steps_per_second": 0.4087171378681868, "throughput/raw_tokens_per_second": 428561.62193481467, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 320, "stop_at": 954, "target": 128747, "training/loss": 8.754878564178943, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.754878564178943, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.524508779264214, "training/generator_lr": 0.0004999993394162303, "training/grad_norm": 0.12473094463348389, "training/positions": 335544320, "training/nonpadding_tokens": 335538862, "throughput/steps_per_second": 0.40904418284154637, "throughput/raw_tokens_per_second": 428893.70628462493, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 330, "stop_at": 954, "target": 128747, "training/loss": 8.726928801089525, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.726928801089525, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.699445743624582, "training/generator_lr": 0.0004999991930265556, "training/grad_norm": 0.0908852145075798, "training/positions": 346030080, "training/nonpadding_tokens": 346024551, "throughput/steps_per_second": 0.40904800709458533, "throughput/raw_tokens_per_second": 428915.01884636155, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 340, "stop_at": 954, "target": 128747, "training/loss": 8.706380737572909, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.706380737572909, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.82765141617893, "training/generator_lr": 0.000499999031997947, "training/grad_norm": 0.1375385820865631, "training/positions": 356515840, "training/nonpadding_tokens": 356510227, "throughput/steps_per_second": 0.4090061161624415, "throughput/raw_tokens_per_second": 428870.56160977244, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 350, "stop_at": 954, "target": 128747, "training/loss": 8.70576949045062, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.70576949045062, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.8153741638795986, "training/generator_lr": 0.0004999988563304143, "training/grad_norm": 0.12410085648298264, "training/positions": 367001600, "training/nonpadding_tokens": 366995799, "throughput/steps_per_second": 0.40923725987019566, "throughput/raw_tokens_per_second": 429108.6753451647, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 360, "stop_at": 954, "target": 128747, "training/loss": 8.678820344060659, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.678820344060659, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.9796571253135453, "training/generator_lr": 0.000499998666023968, "training/grad_norm": 0.09511373192071915, "training/positions": 377487360, "training/nonpadding_tokens": 377481238, "throughput/steps_per_second": 0.4091258551817619, "throughput/raw_tokens_per_second": 428986.4197831198, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 370, "stop_at": 954, "target": 128747, "training/loss": 8.670443738251924, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.670443738251924, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.045591424540134, "training/generator_lr": 0.0004999984610786194, "training/grad_norm": 0.18459655344486237, "training/positions": 387973120, "training/nonpadding_tokens": 387966997, "throughput/steps_per_second": 0.40907723861253203, "throughput/raw_tokens_per_second": 428948.5336476505, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 380, "stop_at": 954, "target": 128747, "training/loss": 8.654626427590847, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.654626427590847, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.126290107650502, "training/generator_lr": 0.0004999982414943807, "training/grad_norm": 0.09897475689649582, "training/positions": 398458880, "training/nonpadding_tokens": 398452694, "throughput/steps_per_second": 0.40907886517110287, "throughput/raw_tokens_per_second": 428947.70292880374, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 390, "stop_at": 954, "target": 128747, "training/loss": 8.644327249377966, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.644327249377966, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.198583494460703, "training/generator_lr": 0.000499998007271265, "training/grad_norm": 0.17077597975730896, "training/positions": 408944640, "training/nonpadding_tokens": 408938400, "throughput/steps_per_second": 0.40892889570047186, "throughput/raw_tokens_per_second": 428790.81752198125, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 400, "stop_at": 954, "target": 128747, "training/loss": 8.631650238484145, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.631650238484145, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.277098792851171, "training/generator_lr": 0.0004999977584092864, "training/grad_norm": 0.11580249667167664, "training/positions": 419430400, "training/nonpadding_tokens": 419424058, "throughput/steps_per_second": 0.40909368040483113, "throughput/raw_tokens_per_second": 428961.6422686361, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 410, "stop_at": 954, "target": 128747, "training/loss": 8.612663938105106, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.612663938105106, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.393990711224917, "training/generator_lr": 0.0004999974949084598, "training/grad_norm": 0.13520166277885437, "training/positions": 429916160, "training/nonpadding_tokens": 429909611, "throughput/steps_per_second": 0.40885654883759254, "throughput/raw_tokens_per_second": 428708.7012233665, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 420, "stop_at": 954, "target": 128747, "training/loss": 8.61954056546092, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.61954056546092, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.3708634902801, "training/generator_lr": 0.0004999972167688007, "training/grad_norm": 0.15599428117275238, "training/positions": 440401920, "training/nonpadding_tokens": 440395082, "throughput/steps_per_second": 0.40897724406762287, "throughput/raw_tokens_per_second": 428831.90323309816, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 430, "stop_at": 954, "target": 128747, "training/loss": 8.598354217410087, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.598354217410087, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.482114862040134, "training/generator_lr": 0.000499996923990326, "training/grad_norm": 0.15363338589668274, "training/positions": 450887680, "training/nonpadding_tokens": 450880746, "throughput/steps_per_second": 0.40917505604906723, "throughput/raw_tokens_per_second": 429047.21549116867, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 440, "stop_at": 954, "target": 128747, "training/loss": 8.581800411641598, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.581800411641598, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.585274482650502, "training/generator_lr": 0.0004999966165730531, "training/grad_norm": 0.15540584921836853, "training/positions": 461373440, "training/nonpadding_tokens": 461366347, "throughput/steps_per_second": 0.40911934925477783, "throughput/raw_tokens_per_second": 428986.2257665248, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 450, "stop_at": 954, "target": 128747, "training/loss": 8.573741032928229, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.573741032928229, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.626546496132943, "training/generator_lr": 0.0004999962945170003, "training/grad_norm": 0.18613947927951813, "training/positions": 471859200, "training/nonpadding_tokens": 471852012, "throughput/steps_per_second": 0.4091469820544181, "throughput/raw_tokens_per_second": 429017.81895836396, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 460, "stop_at": 954, "target": 128747, "training/loss": 8.566143275797367, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.566143275797367, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.687565321906354, "training/generator_lr": 0.0004999959578221868, "training/grad_norm": 0.2156212478876114, "training/positions": 482344960, "training/nonpadding_tokens": 482337515, "throughput/steps_per_second": 0.409071422447864, "throughput/raw_tokens_per_second": 428931.9627291345, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 470, "stop_at": 954, "target": 128747, "training/loss": 8.56047316044569, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.56047316044569, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.732621106814381, "training/generator_lr": 0.000499995606488633, "training/grad_norm": 0.14393538236618042, "training/positions": 492830720, "training/nonpadding_tokens": 492823060, "throughput/steps_per_second": 0.4091828948175741, "throughput/raw_tokens_per_second": 429050.565683994, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 480, "stop_at": 954, "target": 128747, "training/loss": 8.545621962845326, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.545621962845326, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.83139926055602, "training/generator_lr": 0.0004999952405163596, "training/grad_norm": 0.15548564493656158, "training/positions": 503316480, "training/nonpadding_tokens": 503308713, "throughput/steps_per_second": 0.4091654631167813, "throughput/raw_tokens_per_second": 429036.70658268675, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 490, "stop_at": 954, "target": 128747, "training/loss": 8.534178778529167, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.534178778529167, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.913188819502508, "training/generator_lr": 0.0004999948599053887, "training/grad_norm": 0.15385407209396362, "training/positions": 513802240, "training/nonpadding_tokens": 513794206, "throughput/steps_per_second": 0.4091965470146508, "throughput/raw_tokens_per_second": 429062.75293462916, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 500, "stop_at": 954, "target": 128747, "training/loss": 8.532354607433081, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.532354607433081, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.9142992919105355, "training/generator_lr": 0.0004999944646557427, "training/grad_norm": 0.17138919234275818, "training/positions": 524288000, "training/nonpadding_tokens": 524279876, "throughput/steps_per_second": 0.40913950559903667, "throughput/raw_tokens_per_second": 429010.18396746507, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 510, "stop_at": 954, "target": 128747, "training/loss": 8.519752291589976, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.519752291589976, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.992246289715719, "training/generator_lr": 0.0004999940547674456, "training/grad_norm": 0.23349744081497192, "training/positions": 534773760, "training/nonpadding_tokens": 534765323, "throughput/steps_per_second": 0.4092498402357639, "throughput/raw_tokens_per_second": 429116.7509550569, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 520, "stop_at": 954, "target": 128747, "training/loss": 8.511002310365438, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.511002310365438, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.077915969899665, "training/generator_lr": 0.0004999936302405217, "training/grad_norm": 0.16636767983436584, "training/positions": 545259520, "training/nonpadding_tokens": 545250857, "throughput/steps_per_second": 0.40923757937995164, "throughput/raw_tokens_per_second": 429107.45526661817, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 530, "stop_at": 954, "target": 128747, "training/loss": 8.50705091357231, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.50705091357231, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.103381715091973, "training/generator_lr": 0.0004999931910749963, "training/grad_norm": 0.22393527626991272, "training/positions": 555745280, "training/nonpadding_tokens": 555736516, "throughput/steps_per_second": 0.4091472577645206, "throughput/raw_tokens_per_second": 429017.8625703865, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 540, "stop_at": 954, "target": 128747, "training/loss": 8.50012828707695, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.50012828707695, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.162498040342809, "training/generator_lr": 0.0004999927372708958, "training/grad_norm": 0.16262026131153107, "training/positions": 566231040, "training/nonpadding_tokens": 566221693, "throughput/steps_per_second": 0.4092109445533431, "throughput/raw_tokens_per_second": 429064.91839789884, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 550, "stop_at": 954, "target": 128747, "training/loss": 8.485838788002729, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.485838788002729, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.25310115750418, "training/generator_lr": 0.0004999922688282472, "training/grad_norm": 0.22948366403579712, "training/positions": 576716800, "training/nonpadding_tokens": 576707372, "throughput/steps_per_second": 0.4091298984989042, "throughput/raw_tokens_per_second": 429000.4784962091, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 560, "stop_at": 954, "target": 128747, "training/loss": 8.479960573464632, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.479960573464632, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.297951178407191, "training/generator_lr": 0.0004999917857470785, "training/grad_norm": 0.15211758017539978, "training/positions": 587202560, "training/nonpadding_tokens": 587192883, "throughput/steps_per_second": 0.40900110950443674, "throughput/raw_tokens_per_second": 428858.56327209756, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 570, "stop_at": 954, "target": 128747, "training/loss": 8.469613495469094, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.469613495469094, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.360356461642977, "training/generator_lr": 0.0004999912880274186, "training/grad_norm": 0.16650056838989258, "training/positions": 597688320, "training/nonpadding_tokens": 597678457, "throughput/steps_per_second": 0.4090618506550601, "throughput/raw_tokens_per_second": 428924.83056205814, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 580, "stop_at": 954, "target": 128747, "training/loss": 8.464067248255015, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.464067248255015, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.423978365384615, "training/generator_lr": 0.0004999907756692973, "training/grad_norm": 0.1933707296848297, "training/positions": 608174080, "training/nonpadding_tokens": 608164023, "throughput/steps_per_second": 0.4092350995865209, "throughput/raw_tokens_per_second": 429106.16462310374, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 590, "stop_at": 954, "target": 128747, "training/loss": 8.47042583823204, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.47042583823204, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.397513195025083, "training/generator_lr": 0.0004999902486727452, "training/grad_norm": 0.19290679693222046, "training/positions": 618659840, "training/nonpadding_tokens": 618649723, "throughput/steps_per_second": 0.40923986685550484, "throughput/raw_tokens_per_second": 429116.64718867675, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 600, "stop_at": 954, "target": 128747, "training/loss": 8.451304657012225, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.451304657012225, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.502776181020067, "training/generator_lr": 0.0004999897070377936, "training/grad_norm": 0.14480124413967133, "training/positions": 629145600, "training/nonpadding_tokens": 629135483, "throughput/steps_per_second": 0.40929322195290685, "throughput/raw_tokens_per_second": 429175.04950249125, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 610, "stop_at": 954, "target": 128747, "training/loss": 8.44464293718338, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.44464293718338, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.5708073134406355, "training/generator_lr": 0.0004999891507644751, "training/grad_norm": 0.3166012763977051, "training/positions": 639631360, "training/nonpadding_tokens": 639621155, "throughput/steps_per_second": 0.4091030384379982, "throughput/raw_tokens_per_second": 428972.0275264242, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 620, "stop_at": 954, "target": 128747, "training/loss": 8.436087684333325, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.436087684333325, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.608579705790134, "training/generator_lr": 0.0004999885798528228, "training/grad_norm": 0.17485766112804413, "training/positions": 650117120, "training/nonpadding_tokens": 650106618, "throughput/steps_per_second": 0.4092073486197359, "throughput/raw_tokens_per_second": 429072.8513280342, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 630, "stop_at": 954, "target": 128747, "training/loss": 8.42601277679205, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.42601277679205, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.693419797763378, "training/generator_lr": 0.0004999879943028709, "training/grad_norm": 0.1830541342496872, "training/positions": 660602880, "training/nonpadding_tokens": 660592301, "throughput/steps_per_second": 0.409154719955319, "throughput/raw_tokens_per_second": 429026.66914052493, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 640, "stop_at": 954, "target": 128747, "training/loss": 8.422753217816354, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.422753217816354, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.717995531981606, "training/generator_lr": 0.0004999873941146543, "training/grad_norm": 0.2844255864620209, "training/positions": 671088640, "training/nonpadding_tokens": 671078026, "throughput/steps_per_second": 0.40924564228983107, "throughput/raw_tokens_per_second": 429123.7262499539, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 650, "stop_at": 954, "target": 128747, "training/loss": 8.41614634618163, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.41614634618163, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.770396765259197, "training/generator_lr": 0.0004999867792882088, "training/grad_norm": 0.1553221046924591, "training/positions": 681574400, "training/nonpadding_tokens": 681563648, "throughput/steps_per_second": 0.4093646113068772, "throughput/raw_tokens_per_second": 429244.257434084, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 660, "stop_at": 954, "target": 128747, "training/loss": 8.402774673700332, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.402774673700332, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.8563979541178925, "training/generator_lr": 0.0004999861498235714, "training/grad_norm": 0.15722794830799103, "training/positions": 692060160, "training/nonpadding_tokens": 692049393, "throughput/steps_per_second": 0.4092011433274176, "throughput/raw_tokens_per_second": 429077.88426397525, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 670, "stop_at": 954, "target": 128747, "training/loss": 8.388637253642083, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.388637253642083, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.953275240384615, "training/generator_lr": 0.0004999855057207795, "training/grad_norm": 0.30821430683135986, "training/positions": 702545920, "training/nonpadding_tokens": 702535094, "throughput/steps_per_second": 0.4091709109353163, "throughput/raw_tokens_per_second": 429044.38299653574, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 680, "stop_at": 954, "target": 128747, "training/loss": 8.392153864353896, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.392153864353896, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.929978182483278, "training/generator_lr": 0.0004999848469798716, "training/grad_norm": 0.2453027218580246, "training/positions": 713031680, "training/nonpadding_tokens": 713020784, "throughput/steps_per_second": 0.40933680746789036, "throughput/raw_tokens_per_second": 429217.88686979836, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 690, "stop_at": 954, "target": 128747, "training/loss": 8.389953873306514, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.389953873306514, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.976471049331104, "training/generator_lr": 0.0004999841736008871, "training/grad_norm": 0.18437980115413666, "training/positions": 723517440, "training/nonpadding_tokens": 723506441, "throughput/steps_per_second": 0.40930303971969956, "throughput/raw_tokens_per_second": 429181.1283558146, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 700, "stop_at": 954, "target": 128747, "training/loss": 8.373191752284765, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.373191752284765, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.08065295777592, "training/generator_lr": 0.0004999834855838663, "training/grad_norm": 0.22483088076114655, "training/positions": 734003200, "training/nonpadding_tokens": 733992123, "throughput/steps_per_second": 0.4092312249428552, "throughput/raw_tokens_per_second": 429106.8489221248, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 710, "stop_at": 954, "target": 128747, "training/loss": 8.372330252081156, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.372330252081156, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.109221493520067, "training/generator_lr": 0.0004999827829288502, "training/grad_norm": 0.30003583431243896, "training/positions": 744488960, "training/nonpadding_tokens": 744477778, "throughput/steps_per_second": 0.4093183601981357, "throughput/raw_tokens_per_second": 429197.1110203383, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 720, "stop_at": 954, "target": 128747, "training/loss": 8.363162723183631, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.363162723183631, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.140623366952341, "training/generator_lr": 0.0004999820656358808, "training/grad_norm": 0.2820779085159302, "training/positions": 754974720, "training/nonpadding_tokens": 754963460, "throughput/steps_per_second": 0.4090993166694366, "throughput/raw_tokens_per_second": 428968.53410130116, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 730, "stop_at": 954, "target": 128747, "training/loss": 8.363611906021834, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.363611906021834, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.17256414611204, "training/generator_lr": 0.000499981333705001, "training/grad_norm": 0.17488700151443481, "training/positions": 765460480, "training/nonpadding_tokens": 765449028, "throughput/steps_per_second": 0.4091599236355162, "throughput/raw_tokens_per_second": 429027.4202155012, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 740, "stop_at": 954, "target": 128747, "training/loss": 8.350775718688965, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.350775718688965, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.267040852320235, "training/generator_lr": 0.0004999805871362545, "training/grad_norm": 0.22609943151474, "training/positions": 775946240, "training/nonpadding_tokens": 775934485, "throughput/steps_per_second": 0.40918077226016875, "throughput/raw_tokens_per_second": 429044.7392760792, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 750, "stop_at": 954, "target": 128747, "training/loss": 8.339639697223902, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.339639697223902, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.3236831103678925, "training/generator_lr": 0.000499979825929686, "training/grad_norm": 0.17672383785247803, "training/positions": 786432000, "training/nonpadding_tokens": 786420144, "throughput/steps_per_second": 0.4093643800973939, "throughput/raw_tokens_per_second": 429245.52964476595, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 760, "stop_at": 954, "target": 128747, "training/loss": 8.341326329857111, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.341326329857111, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.354978835702341, "training/generator_lr": 0.0004999790500853408, "training/grad_norm": 0.274244099855423, "training/positions": 796917760, "training/nonpadding_tokens": 796905882, "throughput/steps_per_second": 0.40929869585648787, "throughput/raw_tokens_per_second": 429179.88884928176, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 770, "stop_at": 954, "target": 128747, "training/loss": 8.340317272394895, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.340317272394895, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.3556924775292645, "training/generator_lr": 0.0004999782596032654, "training/grad_norm": 0.1526513546705246, "training/positions": 807403520, "training/nonpadding_tokens": 807391508, "throughput/steps_per_second": 0.40932875458747153, "throughput/raw_tokens_per_second": 429206.8231650011, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 780, "stop_at": 954, "target": 128747, "training/loss": 8.323338445276022, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.323338445276022, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.458089464882943, "training/generator_lr": 0.000499977454483507, "training/grad_norm": 0.20543967187404633, "training/positions": 817889280, "training/nonpadding_tokens": 817877191, "throughput/steps_per_second": 0.40928246131017665, "throughput/raw_tokens_per_second": 429160.61467582773, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 790, "stop_at": 954, "target": 128747, "training/loss": 8.330054011195898, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.330054011195898, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.430244369251672, "training/generator_lr": 0.0004999766347261137, "training/grad_norm": 0.22034521400928497, "training/positions": 828375040, "training/nonpadding_tokens": 828362696, "throughput/steps_per_second": 0.40924190705703006, "throughput/raw_tokens_per_second": 429110.8062656024, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 800, "stop_at": 954, "target": 128747, "training/loss": 8.323832194507123, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.323832194507123, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.485776154891305, "training/generator_lr": 0.0004999758003311345, "training/grad_norm": 0.25817862153053284, "training/positions": 838860800, "training/nonpadding_tokens": 838848410, "throughput/steps_per_second": 0.40928149017738297, "throughput/raw_tokens_per_second": 429160.8651493847, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 810, "stop_at": 954, "target": 128747, "training/loss": 8.315256363153457, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.315256363153457, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.552758870714883, "training/generator_lr": 0.0004999749512986192, "training/grad_norm": 0.218985915184021, "training/positions": 849346560, "training/nonpadding_tokens": 849334036, "throughput/steps_per_second": 0.4093218547591466, "throughput/raw_tokens_per_second": 429199.5882630732, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 820, "stop_at": 954, "target": 128747, "training/loss": 8.306205031648279, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.306205031648279, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.62062016617893, "training/generator_lr": 0.0004999740876286187, "training/grad_norm": 0.26127687096595764, "training/positions": 859832320, "training/nonpadding_tokens": 859819502, "throughput/steps_per_second": 0.40917429395211635, "throughput/raw_tokens_per_second": 429038.31473089213, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 830, "stop_at": 954, "target": 128747, "training/loss": 8.306372378021479, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.306372378021479, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.6381901259406355, "training/generator_lr": 0.0004999732093211843, "training/grad_norm": 0.23748648166656494, "training/positions": 870318080, "training/nonpadding_tokens": 870305154, "throughput/steps_per_second": 0.4092527417241979, "throughput/raw_tokens_per_second": 429128.1829765819, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 840, "stop_at": 954, "target": 128747, "training/loss": 8.29961592927575, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.29961592927575, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.693166675376254, "training/generator_lr": 0.0004999723163763688, "training/grad_norm": 0.2031191736459732, "training/positions": 880803840, "training/nonpadding_tokens": 880790680, "throughput/steps_per_second": 0.40944199240633405, "throughput/raw_tokens_per_second": 429321.4656868418, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 850, "stop_at": 954, "target": 128747, "training/loss": 8.29398488998413, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.29398488998413, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.708626737562709, "training/generator_lr": 0.0004999714087942253, "training/grad_norm": 0.22629296779632568, "training/positions": 891289600, "training/nonpadding_tokens": 891276364, "throughput/steps_per_second": 0.40927898094010473, "throughput/raw_tokens_per_second": 429157.0061979961, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 860, "stop_at": 954, "target": 128747, "training/loss": 8.284341107308865, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.284341107308865, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.795767793687291, "training/generator_lr": 0.0004999704865748083, "training/grad_norm": 0.20574678480625153, "training/positions": 901775360, "training/nonpadding_tokens": 901761946, "throughput/steps_per_second": 0.4094457259278737, "throughput/raw_tokens_per_second": 429327.67337662453, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 870, "stop_at": 954, "target": 128747, "training/loss": 8.292009130865335, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.292009130865335, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.777637698578595, "training/generator_lr": 0.0004999695497181725, "training/grad_norm": 0.202379047870636, "training/positions": 912261120, "training/nonpadding_tokens": 912247625, "throughput/steps_per_second": 0.409328868169343, "throughput/raw_tokens_per_second": 429209.1117057048, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 880, "stop_at": 954, "target": 128747, "training/loss": 8.282498709857464, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.282498709857464, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.840077275815218, "training/generator_lr": 0.0004999685982243744, "training/grad_norm": 0.3429924249649048, "training/positions": 922746880, "training/nonpadding_tokens": 922733261, "throughput/steps_per_second": 0.40943453302632893, "throughput/raw_tokens_per_second": 429318.1479144064, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 890, "stop_at": 954, "target": 128747, "training/loss": 8.287351156026125, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.287351156026125, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.816794915342809, "training/generator_lr": 0.0004999676320934704, "training/grad_norm": 0.1516335904598236, "training/positions": 933232640, "training/nonpadding_tokens": 933218837, "throughput/steps_per_second": 0.4094868317085001, "throughput/raw_tokens_per_second": 429370.5294878688, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 900, "stop_at": 954, "target": 128747, "training/loss": 8.270814752578735, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.270814752578735, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.926107859531773, "training/generator_lr": 0.0004999666513255184, "training/grad_norm": 0.21166975796222687, "training/positions": 943718400, "training/nonpadding_tokens": 943704386, "throughput/steps_per_second": 0.4092797105615343, "throughput/raw_tokens_per_second": 429152.2459798785, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 910, "stop_at": 954, "target": 128747, "training/loss": 8.266571156680584, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.266571156680584, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.94475236465301, "training/generator_lr": 0.000499965655920577, "training/grad_norm": 0.2676384150981903, "training/positions": 954204160, "training/nonpadding_tokens": 954190088, "throughput/steps_per_second": 0.40948542781703595, "throughput/raw_tokens_per_second": 429374.216943195, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 920, "stop_at": 954, "target": 128747, "training/loss": 8.263330242037773, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.263330242037773, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.004203464673913, "training/generator_lr": 0.0004999646458787058, "training/grad_norm": 0.21519799530506134, "training/positions": 964689920, "training/nonpadding_tokens": 964675810, "throughput/steps_per_second": 0.409409437332112, "throughput/raw_tokens_per_second": 429295.3544040948, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 930, "stop_at": 954, "target": 128747, "training/loss": 8.25900956839323, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.25900956839323, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.038980847617057, "training/generator_lr": 0.0004999636211999648, "training/grad_norm": 0.17417152225971222, "training/positions": 975175680, "training/nonpadding_tokens": 975161320, "throughput/steps_per_second": 0.4093959066354175, "throughput/raw_tokens_per_second": 429272.48729847366, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 940, "stop_at": 954, "target": 128747, "training/loss": 8.252974602580071, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.252974602580071, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.0835581495610365, "training/generator_lr": 0.0004999625818844157, "training/grad_norm": 0.17934155464172363, "training/positions": 985661440, "training/nonpadding_tokens": 985647032, "throughput/steps_per_second": 0.4092302643027272, "throughput/raw_tokens_per_second": 429107.0693162278, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 950, "stop_at": 954, "target": 128747, "training/loss": 8.25698967576027, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.25698967576027, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.081216359218227, "training/generator_lr": 0.0004999615279321202, "training/grad_norm": 0.30399030447006226, "training/positions": 996147200, "training/nonpadding_tokens": 996132624, "throughput/steps_per_second": 0.40931891305256246, "throughput/raw_tokens_per_second": 429195.11201526446, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 954, "stop_at": 954, "target": 128747, "training/loss": 8.263838732615113, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.263838732615113, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.037092227999582, "training/generator_lr": 0.0004999611022529271, "training/grad_norm": 0.273471862077713, "training/positions": 1000341504, "training/nonpadding_tokens": 1000326925, "throughput/steps_per_second": 0.40920749439312515, "throughput/raw_tokens_per_second": 429084.8507351448, "throughput/peak_reserved_gib": 24.275390625}
+{"validation_step": 954, "loss": 7.030140509605408, "reconstruction_loss": 0.0, "vq_loss": 0.0, "ncp_loss": 7.030140509605408, "mstok_loss": 0.0, "semantic_loss": 0.0, "ncp_accuracy": 17.64464882943144}
+wandb: - 0.022 MB of 0.022 MB uploaded
+wandb: 
+wandb: Run summary:
+wandb:     throughput/peak_reserved_gib 24.27539
+wandb: throughput/raw_tokens_per_second 429084.85074
+wandb:      throughput/steps_per_second 0.40921
+wandb:            training/generator_lr 0.0005
+wandb:               training/grad_norm 0.27347
+wandb:                    training/loss 8.26384
+wandb:              training/mstok_loss 0.0
+wandb:            training/ncp_accuracy 7.03709
+wandb:                training/ncp_loss 8.26384
+wandb:       training/nonpadding_tokens 1000326925
+wandb:               training/positions 1000341504
+wandb:     training/reconstruction_loss 0.0
+wandb:           training/semantic_loss 0.0
+wandb:                 training/vq_loss 0.0
+wandb:                  validation/loss 7.03014
+wandb:            validation/mstok_loss 0.0
+wandb:          validation/ncp_accuracy 17.64465
+wandb:              validation/ncp_loss 7.03014
+wandb:   validation/reconstruction_loss 0.0
+wandb:         validation/semantic_loss 0.0
+wandb:               validation/vq_loss 0.0
+wandb: 
+wandb:  View run iclr-debug-two-stage-135b-20260916-generator at: https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-two-stage-135b-20260916-generator
+wandb:  View project at: https://wandb.ai/mstok/iclr-debug
+wandb: Synced 6 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20260916_151853-iclr-debug-two-stage-135b-20260916-generator/logs
+wandb: WARNING wandb version 0.30.0 is available!  To upgrade, please run:
+wandb: WARNING  $ pip install wandb --upgrade
+wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.

iclr-debug-two-stage-135b-20260916/generator/stopped-20260917/validation-curve.json

@@ -0,0 +1,492 @@
+[
+  {
+    "validation_step": 954,
+    "loss": 7.030140509605408,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 7.030140509605408,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 17.64464882943144
+  },
+  {
+    "validation_step": 1000,
+    "loss": 6.977545759677887,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.977545759677887,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 18.076421404682275
+  },
+  {
+    "validation_step": 2000,
+    "loss": 6.475357372760773,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.475357372760773,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 23.616304347826087
+  },
+  {
+    "validation_step": 3000,
+    "loss": 6.249391813278199,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.249391813278199,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 26.47984949832776
+  },
+  {
+    "validation_step": 4000,
+    "loss": 6.114818637371063,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.114818637371063,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 28.07140468227425
+  },
+  {
+    "validation_step": 4769,
+    "loss": 6.046309206485748,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.046309206485748,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 28.982274247491638
+  },
+  {
+    "validation_step": 5000,
+    "loss": 6.033136506080628,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.033136506080628,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 29.018478260869564
+  },
+  {
+    "validation_step": 6000,
+    "loss": 5.966629896163941,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.966629896163941,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 29.943729096989966
+  },
+  {
+    "validation_step": 7000,
+    "loss": 5.918366656303406,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.918366656303406,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 30.4994983277592
+  },
+  {
+    "validation_step": 8000,
+    "loss": 5.8785524606704715,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.8785524606704715,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 31.012709030100336
+  },
+  {
+    "validation_step": 9000,
+    "loss": 5.842954273223877,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.842954273223877,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 31.484698996655517
+  },
+  {
+    "validation_step": 10000,
+    "loss": 5.827896971702576,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.827896971702576,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 31.649916387959866
+  },
+  {
+    "validation_step": 11000,
+    "loss": 5.794131059646607,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.794131059646607,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.15785953177257
+  },
+  {
+    "validation_step": 12000,
+    "loss": 5.785179510116577,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.785179510116577,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.229933110367895
+  },
+  {
+    "validation_step": 12875,
+    "loss": 5.766966290473938,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.766966290473938,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.42675585284281
+  },
+  {
+    "validation_step": 13000,
+    "loss": 5.763078744411469,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.763078744411469,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.53210702341137
+  },
+  {
+    "validation_step": 14000,
+    "loss": 5.748985006809234,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.748985006809234,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.768645484949836
+  },
+  {
+    "validation_step": 15000,
+    "loss": 5.741684236526489,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.741684236526489,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.87098662207358
+  },
+  {
+    "validation_step": 16000,
+    "loss": 5.729868891239167,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.729868891239167,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.96479933110368
+  },
+  {
+    "validation_step": 17000,
+    "loss": 5.731239960193634,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.731239960193634,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.01923076923077
+  },
+  {
+    "validation_step": 18000,
+    "loss": 5.706959030628204,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.706959030628204,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.36329431438127
+  },
+  {
+    "validation_step": 19000,
+    "loss": 5.7004749608039855,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.7004749608039855,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.474916387959865
+  },
+  {
+    "validation_step": 20000,
+    "loss": 5.6975271821022035,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6975271821022035,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.45827759197324
+  },
+  {
+    "validation_step": 21000,
+    "loss": 5.6820241522789,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6820241522789,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.54824414715719
+  },
+  {
+    "validation_step": 22000,
+    "loss": 5.676759459972382,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.676759459972382,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.63929765886287
+  },
+  {
+    "validation_step": 23000,
+    "loss": 5.670730285644531,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.670730285644531,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.70091973244147
+  },
+  {
+    "validation_step": 24000,
+    "loss": 5.658136134147644,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.658136134147644,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.9554347826087
+  },
+  {
+    "validation_step": 25000,
+    "loss": 5.6540408921241765,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6540408921241765,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.90209030100335
+  },
+  {
+    "validation_step": 25750,
+    "loss": 5.651979362964631,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.651979362964631,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.96053511705686
+  },
+  {
+    "validation_step": 26000,
+    "loss": 5.649268729686737,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.649268729686737,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.95083612040134
+  },
+  {
+    "validation_step": 27000,
+    "loss": 5.640766968727112,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.640766968727112,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.143478260869564
+  },
+  {
+    "validation_step": 28000,
+    "loss": 5.643126902580262,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.643126902580262,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.08010033444816
+  },
+  {
+    "validation_step": 29000,
+    "loss": 5.642015235424042,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.642015235424042,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.197909698996654
+  },
+  {
+    "validation_step": 30000,
+    "loss": 5.632075030803681,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.632075030803681,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.23210702341137
+  },
+  {
+    "validation_step": 31000,
+    "loss": 5.632021889686585,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.632021889686585,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.24314381270903
+  },
+  {
+    "validation_step": 32000,
+    "loss": 5.6199514770507815,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6199514770507815,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.39958193979933
+  },
+  {
+    "validation_step": 33000,
+    "loss": 5.614811289310455,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.614811289310455,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.48377926421405
+  },
+  {
+    "validation_step": 34000,
+    "loss": 5.617471935749054,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.617471935749054,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.44857859531773
+  },
+  {
+    "validation_step": 35000,
+    "loss": 5.614271013736725,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.614271013736725,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.45108695652174
+  },
+  {
+    "validation_step": 36000,
+    "loss": 5.609555795192718,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.609555795192718,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.55819397993311
+  },
+  {
+    "validation_step": 37000,
+    "loss": 5.612579424381257,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.612579424381257,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.4626254180602
+  },
+  {
+    "validation_step": 38000,
+    "loss": 5.6007399559021,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6007399559021,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.687207357859535
+  },
+  {
+    "validation_step": 39000,
+    "loss": 5.611569912433624,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.611569912433624,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.53386287625418
+  },
+  {
+    "validation_step": 40000,
+    "loss": 5.595582022666931,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.595582022666931,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.729180602006686
+  },
+  {
+    "validation_step": 41000,
+    "loss": 5.603518695831299,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.603518695831299,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.72182274247491
+  },
+  {
+    "validation_step": 42000,
+    "loss": 5.593213293552399,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.593213293552399,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.79916387959866
+  },
+  {
+    "validation_step": 43000,
+    "loss": 5.594714949131012,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.594714949131012,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.682441471571906
+  },
+  {
+    "validation_step": 44000,
+    "loss": 5.592791588306427,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.592791588306427,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.8190635451505
+  },
+  {
+    "validation_step": 45000,
+    "loss": 5.5970091938972475,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.5970091938972475,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 34.62951505016722
+  }
+]