Archive stopped generator: latest 39k, best validation 38k, results and best Gen-PPL provenance

iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-38000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b4a9a20ced37f2a1c0277eaf24ade63d45bf232ad331e6693c5cb9811eaed0c
+size 2565487243

iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-39000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e2cb87f576a82fb45d1a91351c7e25d87809f53401af6289e1ca5837bf1bf90
+size 2565487243

iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp-config.yaml

@@ -0,0 +1,179 @@
+task: next-concept
+experiment: iclr-debug-generator-decay27b-20260917-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000
+model:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: gpt2
+    special_tokens:
+      eos_token: <|endoftext|>
+      pad_token: <|pad|>
+  vqvae:
+    n_layers: 6
+    context_length: 256
+    embed_dim: 384
+    in_vocab_size: 50304
+    vqvae_vocab_size:
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    compression_factor: 1
+    dropout: 0.1
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: 0.2
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 4
+        - 9
+        - 16
+        - 25
+        - 36
+        - 49
+        - 64
+        - 81
+        - 100
+        - 121
+        - 144
+        - 169
+        - 196
+        - 225
+        - 256
+      aux:
+        fine_drop:
+          prob: 0.5
+          min_keep: 1
+    checkpoint_path: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae.pt
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+wandb:
+  entity: mstok
+  project: owt-repro
+  group: owtsmall-repro-ctx256-sharedhead-ncp
+  enable: false
+  id: owtsmall-repro-ctx256-sharedhead-ncp-5ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0005
+  reset_lr: true
+  warmup_iters: 300
+  lr_decay_iters: null
+  min_lr: 1.0e-05
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 4
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+  level_loss_alpha: 1.0
+training:
+  seed: 55
+  log_interval: 10
+  eval_interval: 5000
+  checkpoint_interval: 2500
+  val_iters: 100
+  total_iters: null
+  tqdm_interval: 10
+  n_epochs: 5
+  batch_size: 96
+  eval_batch_size: 96
+  demo_batch_size: 4
+  keep_last: 8
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true

iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1c534d7a096cfc16620b15634a3ecd38173a790142867102464e522e72af1ae
+size 427469662

iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae-config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-generator-decay27b-20260917-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d606945a7150c287b927928f6ad35ffa07db7670117a3fc7ccfbbc85d725ed
+size 1300864149

iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp-config.yaml

@@ -0,0 +1,179 @@
+task: next-concept
+experiment: iclr-debug-generator-decay27b-20260917-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000
+model:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: gpt2
+    special_tokens:
+      eos_token: <|endoftext|>
+      pad_token: <|pad|>
+  vqvae:
+    n_layers: 6
+    context_length: 256
+    embed_dim: 384
+    in_vocab_size: 50304
+    vqvae_vocab_size:
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    - 16384
+    compression_factor: 1
+    dropout: 0.1
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: 0.2
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 4
+        - 9
+        - 16
+        - 25
+        - 36
+        - 49
+        - 64
+        - 81
+        - 100
+        - 121
+        - 144
+        - 169
+        - 196
+        - 225
+        - 256
+      aux:
+        fine_drop:
+          prob: 0.5
+          min_keep: 1
+    checkpoint_path: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae.pt
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+wandb:
+  entity: mstok
+  project: owt-repro
+  group: owtsmall-repro-ctx256-sharedhead-ncp
+  enable: false
+  id: owtsmall-repro-ctx256-sharedhead-ncp-5ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0005
+  reset_lr: true
+  warmup_iters: 300
+  lr_decay_iters: null
+  min_lr: 1.0e-05
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 4
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+  level_loss_alpha: 1.0
+training:
+  seed: 55
+  log_interval: 10
+  eval_interval: 5000
+  checkpoint_interval: 2500
+  val_iters: 100
+  total_iters: null
+  tqdm_interval: 10
+  n_epochs: 5
+  batch_size: 96
+  eval_batch_size: 96
+  demo_batch_size: 4
+  keep_last: 8
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true

iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:823f76784466ef5ecee3144fdbcb921fe0c992b6473731469bb8b53991cbe4a8
+size 427469662

iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae-config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-generator-decay27b-20260917-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2925de4fc4ad65ce22d9a5be5fd27b9f0d724cb3976f9b72be66546422847e8
+size 1300864149

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/README.md

@@ -0,0 +1,7 @@
+# Stopped shortened-decay generator — September 19, 2026
+
+Stopped at user request to launch the latest-branch no-semantic component ablation. Latest retained resumable checkpoint: step 39,000 (40.894B input positions). Best validation-loss checkpoint: step 38,000. The generator used the frozen final tokenizer at step 128,747, cosine LR decay over 27B positions, then constant minimum LR 1e-5.
+
+Evaluation used FP64 GPT-2 Large scoring with TF32 disabled, five seeds ×128 samples, true level-zero code supplied, temperature 1, top-k 0, top-p 1. Step 12,875 (13.5B) scored 190.644804 ± 4.998820 seed SE. Step 25,750 (27B) scored 191.701261 ± 5.558756. Both evaluated checkpoints and complete sample/result files are backed up and SHA-256 verified. No generation-PPL claim is made for steps 38,000 or 39,000.
+
+Full resumable states include model, optimizer, scheduler, token counts and per-rank RNG. This archive adds steps 38,000/39,000, standalone exports, stopped-run metadata and training/validation logs, and verifies the earlier evaluated milestones. See backup-manifest.json for hashes.

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/STOPPED.json

@@ -0,0 +1,10 @@
+{
+  "stage": "stopped-by-user",
+  "updated_utc": "2026-09-19T01:12:47.170978+00:00",
+  "last_logged_step": 39050,
+  "latest_resumable_step": 39000,
+  "best_validation_step": 38000,
+  "best_measured_gen_ppl_step": 12875,
+  "best_measured_gen_ppl": 190.64480396208324,
+  "reason": "User requested stopping generator and launching latest-branch no-semantic ablation after tests"
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/STOP_REQUESTED.json

@@ -0,0 +1,26 @@
+{
+  "requested_utc": "2026-09-19T01:12:08.507638+00:00",
+  "reason": "User requested stop, HF archive, then latest-branch no-semantic ablation qualification and production",
+  "retained_checkpoints": [
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-39000.pt",
+      "step": 39000,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-39000.pt"
+    },
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/best_checkpoint.pt",
+      "step": 38000,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-38000.pt"
+    },
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/milestone-iter-12875.pt",
+      "step": 12875,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-12875.pt"
+    },
+    {
+      "source": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/milestone-iter-25750.pt",
+      "step": 25750,
+      "retained": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-25750.pt"
+    }
+  ]
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/backup-manifest.json

@@ -0,0 +1,316 @@
+{
+  "repo": "iskhare/iclr-debug",
+  "run": "iclr-debug-generator-decay27b-20260917",
+  "stopped": {
+    "stage": "stopped-by-user",
+    "updated_utc": "2026-09-19T01:12:47.170978+00:00",
+    "last_logged_step": 39050,
+    "latest_resumable_step": 39000,
+    "best_validation_step": 38000,
+    "best_measured_gen_ppl_step": 12875,
+    "best_measured_gen_ppl": 190.64480396208324,
+    "reason": "User requested stopping generator and launching latest-branch no-semantic ablation after tests"
+  },
+  "files": [
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-38000.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-38000.pt",
+      "size_bytes": 2565487243,
+      "sha256": "3b4a9a20ced37f2a1c0277eaf24ade63d45bf232ad331e6693c5cb9811eaed0c"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp-config.yaml",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp-config.yaml",
+      "size_bytes": 3540,
+      "sha256": "f1f2ece9d3d173445d4fccedbfa4e6018d7e1b2d25e94bc78bc3e7f63525eb79"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/ncp.pt",
+      "size_bytes": 427469662,
+      "sha256": "a1c534d7a096cfc16620b15634a3ecd38173a790142867102464e522e72af1ae"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae-config.yaml",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae-config.yaml",
+      "size_bytes": 2212,
+      "sha256": "889a12893fc47d994b7df9558737cb49fe6856ba041cb2d3ac64cee97f14c3d7"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-38000/vqvae.pt",
+      "size_bytes": 1300864149,
+      "sha256": "04d606945a7150c287b927928f6ad35ffa07db7670117a3fc7ccfbbc85d725ed"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/manual-checkpoints/checkpoint-iter-39000.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-39000.pt",
+      "size_bytes": 2565487243,
+      "sha256": "7e2cb87f576a82fb45d1a91351c7e25d87809f53401af6289e1ca5837bf1bf90"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp-config.yaml",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp-config.yaml",
+      "size_bytes": 3540,
+      "sha256": "a328a1a717e4ac4d0a8542e24bb342d687e86296ab4c22e4e805663d0e664f1c"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/ncp.pt",
+      "size_bytes": 427469662,
+      "sha256": "823f76784466ef5ecee3144fdbcb921fe0c992b6473731469bb8b53991cbe4a8"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae-config.yaml",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae-config.yaml",
+      "size_bytes": 2212,
+      "sha256": "55235186ffb685acf7dc0228609e4ab357cd467dcaa04424c2604d029b5e1410"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/exports/step-39000/vqvae.pt",
+      "size_bytes": 1300864149,
+      "sha256": "a2925de4fc4ad65ce22d9a5be5fd27b9f0d724cb3976f9b72be66546422847e8"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/EVAL_DONE.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/EVAL_DONE.json",
+      "size_bytes": 122,
+      "sha256": "82e92716a726a27245e22c3e329e3d8cec23fdb3e6074f80e16cae7ae27ff59d"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-0.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-0.json",
+      "size_bytes": 138926,
+      "sha256": "783d4deea0a51eb2f3be6aebd6894a62e5b08895e0aca95a028bdd66df8fdf31"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-0.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-0.samples.json",
+      "size_bytes": 139304,
+      "sha256": "fdf51b2650d56d040d9557f1cc136cb775181ef7157168cdd41c9902612a8d9f"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-1.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-1.json",
+      "size_bytes": 141215,
+      "sha256": "e033c2fd5d651c4ed6dc3e54936100ca1598a480573ef87cd62504cc9a493b56"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-1.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-1.samples.json",
+      "size_bytes": 141593,
+      "sha256": "45bc0d246175f1da9372ea4acb1faa709d152af0d991a7f845242b99c8842a94"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-2.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-2.json",
+      "size_bytes": 142061,
+      "sha256": "c2860df04a92f3795fca0682399b4a3477e146a029162fab4f78ba4fdc3cb969"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-2.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-2.samples.json",
+      "size_bytes": 142438,
+      "sha256": "cb1716aa321bb9844fa47761e67469a3fdefe89b1f9b2cea4e4b61b2467d1fc9"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-3.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-3.json",
+      "size_bytes": 137492,
+      "sha256": "c0ce270407b61f5d3e2b8e0993a8d2c18de6b7cfdff208883e00a7e89d38ae87"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-3.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-3.samples.json",
+      "size_bytes": 137869,
+      "sha256": "40e2716cdbd7bd3e98e4b627a68b78b85ada388d359be8df95467677084d3534"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-4.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-4.json",
+      "size_bytes": 142507,
+      "sha256": "19c88b9707267e1d47ae07166bb79ed31e250dfdaf1c4d798985225c36f465fb"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-4.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/random-seed-4.samples.json",
+      "size_bytes": 142884,
+      "sha256": "08a4b7e340784d058de281f2c23684e60b5097e5fa9295f0c6f9eabaa8522a95"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/summary.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/summary.json",
+      "size_bytes": 923,
+      "sha256": "c4f4bdf1a89121ea86084c30dc85cdd5132af435a362e0bd7fa82c7705f3ed6b"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/EVAL_DONE.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/EVAL_DONE.json",
+      "size_bytes": 122,
+      "sha256": "a2f6bc0a9076b28a1b469d432bc48a66196ca33a52bb4ec9d63160ee4be368e2"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-0.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-0.json",
+      "size_bytes": 139139,
+      "sha256": "3db1fde304b76d87d6ae1ad4e4fefe571392ed84cc9a9c3972cd05efae62eb13"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-0.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-0.samples.json",
+      "size_bytes": 139516,
+      "sha256": "2c0337125e81cfecd742c176368b1dd566b20a23895cb2bc8f3350944bb818c4"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-1.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-1.json",
+      "size_bytes": 143085,
+      "sha256": "9105e7ed6a16d1c9f63af7a7a9670108b634c2d2b0a2446da3328b7414d5c132"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-1.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-1.samples.json",
+      "size_bytes": 143461,
+      "sha256": "5ed4aaa0b1aec180eceff09ee29e5cb5d06316d95339c11520a0b7ebfd6e52c6"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-2.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-2.json",
+      "size_bytes": 141906,
+      "sha256": "6c0ce0b6417dc1aacc13952983ff8c07039a9a81dc14a6f25ee899723d3bd232"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-2.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-2.samples.json",
+      "size_bytes": 142283,
+      "sha256": "0e9cd1b8b1ec39df727612251a468805f5198716abe845cac60897cd5838d3a1"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-3.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-3.json",
+      "size_bytes": 137256,
+      "sha256": "2ae0ab67a99f7e8a47161beb5af40afd521cfc7b841fd01eda0c9975fc2bd541"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-3.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-3.samples.json",
+      "size_bytes": 137633,
+      "sha256": "761e4a22ff954bd43b8f786668ae93baebd528f9ea0f7f6daa4def790ee77088"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-4.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-4.json",
+      "size_bytes": 141880,
+      "sha256": "4f2e6d6c62377935663fe82eac130e451c05b8ccc50cdf72776244a12e10b90f"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-4.samples.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/random-seed-4.samples.json",
+      "size_bytes": 142257,
+      "sha256": "cc4258ed80065b209de606b570464e226277e99b0c6b073b839e035cdda56599"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/summary.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-25750/summary.json",
+      "size_bytes": 925,
+      "sha256": "43405eec3d1c2d77ac2043117ca7a014c22e1079631f2c7fe08bada8ca3012bb"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/STOPPED.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/STOPPED.json",
+      "size_bytes": 378,
+      "sha256": "bf002de472202931fdbc84bec688ca7c9d1cf6f366670721f09e87f53b6e46a6"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/STOP_REQUESTED.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/STOP_REQUESTED.json",
+      "size_bytes": 1375,
+      "sha256": "3a598b54db0eba6540b3ac9fe77fe52ce48b26b90da73729ee926588377357c4"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/manifest.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/manifest.json",
+      "size_bytes": 28904,
+      "sha256": "9a3313b4226a6a6ba8fb00d6f18bd4a706d947e85b89a7ef5e9a055662ec6beb"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/git-provenance.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/git-provenance.json",
+      "size_bytes": 59,
+      "sha256": "96f601db3a051fd136086ca5f54c87ce2a72449e528d826a3b882541953e3fa2"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/config.yaml",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/config.yaml",
+      "size_bytes": 4599,
+      "sha256": "5d0c2d02bbb717d1a72076d8900ffe7c006eb7cea1ba1804d151112832596eff"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/progress.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/progress.json",
+      "size_bytes": 404,
+      "sha256": "e6bb36b00206701e083a654bf8f039f5299482f7603c04b89056854584b48917"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/hf-stop-backup-20260919/validation-curve.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/validation-curve.json",
+      "size_bytes": 10508,
+      "sha256": "b68836743d9a03f093e79de89ec770e89df14f295e37061de0d563008367ec03"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/training-to-12875.log",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-12875.log",
+      "size_bytes": 508369,
+      "sha256": "1935391651207e7cf6570ad5e9cd08fffda790632c06aa3cf5514c6c10483e5d"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/training-to-25750.log",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-25750.log",
+      "size_bytes": 804001,
+      "sha256": "0165b978e9a6d0e8e144e2534c8a3bb33e84dc6a4a22c19168d141092919353f"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/training-to-4769.log",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-4769.log",
+      "size_bytes": 242596,
+      "sha256": "8e27af71fa22be379762c52061e07258e5b445da4280af72440dcb687f30d36f"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/training-to-51499.log",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-51499.log",
+      "size_bytes": 809636,
+      "sha256": "6f2f602581d40a4e7d9ca545ed0cab0aaefb80f6e4e7fea1231f58db1c2a6b59"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/training-to-954.log",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-954.log",
+      "size_bytes": 67108,
+      "sha256": "0103f04520204545dc2cc00cfc5f9029b44180fb58d94073df1f6a89dd37cf60"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/milestone-iter-12875.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/milestone-iter-12875.pt",
+      "size_bytes": 2565487499,
+      "sha256": "6c7dd0863eb47a59baba2d618962c677d0f5ed3f95a318a36cae7442be95f781"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/summary.json",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/evaluation/step-12875/summary.json",
+      "size_bytes": 923,
+      "sha256": "c4f4bdf1a89121ea86084c30dc85cdd5132af435a362e0bd7fa82c7705f3ed6b"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/milestone-iter-25750.pt",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/milestone-iter-25750.pt",
+      "size_bytes": 2565487243,
+      "sha256": "d83149925b482859d1d9eaaa1c582595d50d0582cd3672d63a5d1278fe26d0a4"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/hf-stop-backup-20260919/README.md",
+      "repo_path": "iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/README.md",
+      "size_bytes": 1098,
+      "sha256": "e66ae46ecb082b615aeebed748bbc26ca3833db5e1e5ba7a43067f0e1e5f29ab"
+    }
+  ]
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/config.yaml

@@ -0,0 +1,223 @@
+task: mstok-next-concept
+experiment: iclr-debug-generator-decay27b-20260917-generator
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>
+codec:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer_config:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+generator:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: 768
+  context_length: 256
+  vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  prefix_len: 0
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  use_relu2: true
+  use_flex_attention: true
+  shared_output_head: true
+  shared_head_per_level_bias: false
+  shared_head_per_level_scale: false
+  shared_head_adapter_rank: 0
+objectives:
+  codec_weight: 1.0
+  ncp_weight: 1.0
+  soft_assignment_temperature: 1.0
+  prediction_temperature: 1.0
+  mstok_weight: 0.25
+  residual_weight: 1.0
+  reconstruction_weight: 1.0
+  teacher_ema_decay: 0.999
+  teacher_ema_warmup_steps: 1000
+  mstok_warmup_steps: 500
+optimization:
+  codec_lr: 0.001
+  codec_min_lr: 0.0001
+  codec_warmup_iters: 0
+  codec_lr_decay_iters: 25750
+  generator_lr: 0.0005
+  generator_min_lr: 1.0e-05
+  generator_warmup_iters: 225
+  generator_lr_decay_iters: 25750
+  beta_1: 0.9
+  codec_beta_2: 0.99
+  generator_beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  level_loss_alpha: 1.0
+  grad_accumulation_steps: 16
+  corruption:
+    mode: per_level
+    per_level_probs:
+    - 0.85
+    - 0.8321428571
+    - 0.8142857143
+    - 0.7964285714
+    - 0.7785714286
+    - 0.7607142857
+    - 0.7428571429
+    - 0.725
+    - 0.7071428571
+    - 0.6892857143
+    - 0.6714285714
+    - 0.6535714286
+    - 0.6357142857
+    - 0.6178571429
+    - 0.6
+    skip_level0: true
+training:
+  log_interval: 10
+  eval_interval: 1000
+  checkpoint_interval: 1000
+  eval_batch_size: 4
+  val_iters: 25
+  keep_last: 3
+  seed: 55
+  codec_initialization_seed: 42
+  generator_initialization_seed: 55
+  total_iters: 128747
+  batch_size: 32
+  expected_world_size: 8
+  expected_global_batch_size: 4096
+  resume_checkpoint: null
+  milestone_steps:
+  - 954
+  - 4769
+  - 12875
+  - 25750
+  - 51499
+  - 77248
+  - 102997
+  - 128747
+  generation_eval_steps:
+  - 12875
+  - 25750
+  - 51499
+  - 77248
+  - 102997
+  - 128747
+torch_compile:
+  enable: true
+  scope: loss
+  mode: default
+  dynamic: false
+  fullgraph: true
+  backend: inductor
+mixed_precision:
+  enable: true
+wandb:
+  entity: mstok
+  project: iclr-debug
+  group: full-owt-ctx256-16sq-135b-v1
+  enable: true
+  id: iclr-debug-generator-decay27b-20260917-generator
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+export:
+  vqvae_config_template: /home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/config/repro-ctx256/vqvae.yaml
+  ncp_config_template: /home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/config/repro-ctx256/ncp-sharedhead.yaml
+iclr_debug:
+  stage: generator
+  target_positions: 135000000000
+  lr_horizon_positions: 27000000000
+  stop_step: 128747
+  pilot: false
+  tokenizer_checkpoint: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt
+  tokenizer_steps: 128747
+  budget_unit: input positions including padding; log non-padding tokens separately

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/git-provenance.json

@@ -0,0 +1,3 @@
+{
+  "commit": "1b5226d4bbdd34653cb5fdd58d436355fb76ac8c"
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/manifest.json

@@ -0,0 +1,517 @@
+{
+  "format": "iclr-debug-production-v1",
+  "variant": "generator",
+  "configs": {
+    "generator": {
+      "task": "mstok-next-concept",
+      "experiment": "iclr-debug-generator-decay27b-20260917-generator",
+      "experiment_dir": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator",
+      "dataset": {
+        "train_source": "/home/ubuntu/data/full_owt/train_gpt2.bin",
+        "validate_source": "/home/ubuntu/data/full_owt/valid_gpt2.bin",
+        "pad_token_id": 50257
+      },
+      "pre_tokenizer": {
+        "name": "hf",
+        "tokenizer": "hf",
+        "model_id": "gpt2",
+        "special_tokens": {
+          "eos_token": "<|endoftext|>",
+          "pad_token": "<|pad|>"
+        }
+      },
+      "codec": {
+        "n_layers": 6,
+        "context_length": 256,
+        "embed_dim": 384,
+        "in_vocab_size": 50304,
+        "vqvae_vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "compression_factor": 1,
+        "dropout": 0.1,
+        "pre_quant_groupnorm": 4,
+        "pre_quant_dropout": 0.2,
+        "vector_quantizer_config": {
+          "clss": "multiscale_residual_vector_quantizer",
+          "decay": 0.99,
+          "epsilon": 1e-05,
+          "commitment_cost": 0.25,
+          "learned_l1_sampling": true,
+          "learned_all_sampling": false,
+          "quant_resi": {
+            "enabled": true,
+            "ratio": 0.5,
+            "share_mode": 0,
+            "learnable_ratio": false
+          },
+          "levels": {
+            "use_manual_levels": true,
+            "manual_levels": [
+              1,
+              4,
+              9,
+              16,
+              25,
+              36,
+              49,
+              64,
+              81,
+              100,
+              121,
+              144,
+              169,
+              196,
+              225,
+              256
+            ]
+          },
+          "aux": {
+            "fine_drop": {
+              "prob": 0.5,
+              "min_keep": 1
+            }
+          }
+        }
+      },
+      "generator": {
+        "n_layer": 12,
+        "n_head": 12,
+        "bias": true,
+        "dropout": 0.1,
+        "n_embd": 768,
+        "context_length": 256,
+        "vocab_size": [
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384,
+          16384
+        ],
+        "attn_pattern": "block_diagonal",
+        "use_positional_encoding": true,
+        "use_level_encoding": true,
+        "prefix_len": 0,
+        "use_rope": true,
+        "rope_base": 10000,
+        "use_qk_norm": true,
+        "post_upsample_conv": {
+          "enabled": true,
+          "kernel_size": 3
+        },
+        "use_relu2": true,
+        "use_flex_attention": true,
+        "shared_output_head": true,
+        "shared_head_per_level_bias": false,
+        "shared_head_per_level_scale": false,
+        "shared_head_adapter_rank": 0
+      },
+      "objectives": {
+        "codec_weight": 1.0,
+        "ncp_weight": 1.0,
+        "soft_assignment_temperature": 1.0,
+        "prediction_temperature": 1.0,
+        "mstok_weight": 0.25,
+        "residual_weight": 1.0,
+        "reconstruction_weight": 1.0,
+        "teacher_ema_decay": 0.999,
+        "teacher_ema_warmup_steps": 1000,
+        "mstok_warmup_steps": 500
+      },
+      "optimization": {
+        "codec_lr": 0.001,
+        "codec_min_lr": 0.0001,
+        "codec_warmup_iters": 0,
+        "codec_lr_decay_iters": 25750,
+        "generator_lr": 0.0005,
+        "generator_min_lr": 1e-05,
+        "generator_warmup_iters": 225,
+        "generator_lr_decay_iters": 25750,
+        "beta_1": 0.9,
+        "codec_beta_2": 0.99,
+        "generator_beta_2": 0.99,
+        "weight_decay": 0.1,
+        "max_grad_norm": 1.0,
+        "level_loss_alpha": 1.0,
+        "grad_accumulation_steps": 16,
+        "corruption": {
+          "mode": "per_level",
+          "per_level_probs": [
+            0.85,
+            0.8321428571,
+            0.8142857143,
+            0.7964285714,
+            0.7785714286,
+            0.7607142857,
+            0.7428571429,
+            0.725,
+            0.7071428571,
+            0.6892857143,
+            0.6714285714,
+            0.6535714286,
+            0.6357142857,
+            0.6178571429,
+            0.6
+          ],
+          "skip_level0": true
+        }
+      },
+      "training": {
+        "log_interval": 10,
+        "eval_interval": 1000,
+        "checkpoint_interval": 1000,
+        "eval_batch_size": 4,
+        "val_iters": 25,
+        "keep_last": 3,
+        "seed": 55,
+        "codec_initialization_seed": 42,
+        "generator_initialization_seed": 55,
+        "total_iters": 128747,
+        "batch_size": 32,
+        "expected_world_size": 8,
+        "expected_global_batch_size": 4096,
+        "resume_checkpoint": null,
+        "milestone_steps": [
+          954,
+          4769,
+          12875,
+          25750,
+          51499,
+          77248,
+          102997,
+          128747
+        ],
+        "generation_eval_steps": [
+          12875,
+          25750,
+          51499,
+          77248,
+          102997,
+          128747
+        ]
+      },
+      "torch_compile": {
+        "enable": true,
+        "scope": "loss",
+        "mode": "default",
+        "dynamic": false,
+        "fullgraph": true,
+        "backend": "inductor"
+      },
+      "mixed_precision": {
+        "enable": true
+      },
+      "wandb": {
+        "entity": "mstok",
+        "project": "iclr-debug",
+        "group": "full-owt-ctx256-16sq-135b-v1",
+        "enable": true,
+        "id": "iclr-debug-generator-decay27b-20260917-generator",
+        "resume": "allow",
+        "gradients_and_params": {
+          "enable": false,
+          "log": "all",
+          "log_freq": 1000
+        }
+      },
+      "export": {
+        "vqvae_config_template": "/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/config/repro-ctx256/vqvae.yaml",
+        "ncp_config_template": "/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/config/repro-ctx256/ncp-sharedhead.yaml"
+      },
+      "iclr_debug": {
+        "stage": "generator",
+        "target_positions": 135000000000,
+        "lr_horizon_positions": 27000000000,
+        "stop_step": 128747,
+        "pilot": false,
+        "tokenizer_checkpoint": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt",
+        "tokenizer_steps": 128747,
+        "budget_unit": "input positions including padding; log non-padding tokens separately"
+      }
+    }
+  },
+  "sources": {
+    "config/repro-ctx256/alignment-generator.yaml": "f62f36d078a3a373a19ea716a3ebc7eda5ae2fb7d670c5b935c734826bb76895",
+    "config/repro-ctx256/alignment-joint.yaml": "bb511950d20335e4237e406464ab26760a7fd9c9c9472c0507addd88d33d5178",
+    "config/repro-ctx256/alignment-tokenizer.yaml": "2d32955296f7d15990f284328c1cf8f52071abdba28033a65e272bb49cf5dd56",
+    "config/repro-ctx256/alignment.yaml": "3a486fa2ed28eca4fc406232669399b0625d274376ac43271f4ccf187b1e9077",
+    "config/repro-ctx256/joint.yaml": "8b4889111687a52e2d7d71995bda45a7f01f2777674c88191f338dca4c1732d2",
+    "config/repro-ctx256/mstok-semantic-eostok-matched10ep.yaml": "ae8a72c0c9f1cb5d2e301d83a1cdc08985a1f895bf627a5f75b1b3ce627f5e58",
+    "config/repro-ctx256/mstok-semantic-eostok.yaml": "2c9c48b21325c0e479990701b28116d78f611101724846747d173e52145fe15a",
+    "config/repro-ctx256/mstok-semantic-gear-matched10ep.yaml": "164cb8ac39fa407bb337f559ad8a9c62a34708fb900b8829d3ccf182961f2d0c",
+    "config/repro-ctx256/mstok-semantic-gear.yaml": "8cb8c998551daef0bf7fd9ec77b13b34c06662028d8820767e1432b5a9e8be54",
+    "config/repro-ctx256/mstok-semantic.yaml": "e0b00dbfa3eb838103f657c19bde20e36375c7fbe692623452ea093003f1ac98",
+    "config/repro-ctx256/mstok-w1-pilot.yaml": "21ee22eeccaaea2563c92ccc84f6df3871ea5da6a5ce980656088ea577dc1438",
+    "config/repro-ctx256/mstok-w1.yaml": "b6bf6079114b3c997fab89e85a53318bb2ec5dae55678513599d84152eab701b",
+    "config/repro-ctx256/mstok.yaml": "d9bb216f4eeb58857d72be399d5f8847d197ce70d0e504879f0788045483dc87",
+    "config/repro-ctx256/ncp-sharedhead.yaml": "86c9efbcb2b4f74b7253aa3bc4393de4a6a05e20ea30da573d1ca094ec600645",
+    "config/repro-ctx256/ncp.yaml": "93663654c4ce5ad0931db052edf507fdcb7c0ff9fc9722f4018820deee5c5acf",
+    "config/repro-ctx256/substitution-generator.yaml": "064f55887b9b2b8f3b0b52c094d4a193510dc1148ad7a5c7f53e317971f87c7c",
+    "config/repro-ctx256/substitution-joint.yaml": "b1576f4ecfe6fc6029a0cd868271b80a1d988b7f3667c6b1a8d24a6a4015df3b",
+    "config/repro-ctx256/substitution-tokenizer.yaml": "e29be5046f0fafbfe4c06973be4cd036a004e845348911cc0e6e05a7fca5d425",
+    "config/repro-ctx256/substitution.yaml": "ebba1c7e664de0e84913bdfd38e794837781c42c1b48cde90135ff9417856a95",
+    "config/repro-ctx256/vqvae.yaml": "05fde46e57c3e6b411e8a6afcac251f178e4b8a0856a0b99332bb284fb4710ce",
+    "data/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/serialized_dataset.py": "5622c2d94236cd1b9b4ca18e8d5c272a86b54d193d99048fb9b7cd2687e8dbc6",
+    "data/tinysentences/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinysentences/clean.py": "918a9a91aae484a5e06d6d212c2c0974946a4032e061c62a7b65851735bfc8dd",
+    "data/tinysentences/clean_parsed.py": "009a087f967e171bc6098484dbfd58635302900f6b07629629898f1d00fa70a3",
+    "data/tinysentences/generate_meta.py": "662714f1370e05567d412add168a7c3a05f4b1c9adb047fad374900bc1936a7e",
+    "data/tinysentences/generate_metadata_char.py": "afd3336ea03ff0653ccaf41c121d9b93debf7a380db6cd881b62b308ee6ee68c",
+    "data/tinysentences/generate_training_data_word.py": "82e2edcb7ea4db0609ccf2b047a0b207e7b8fcff4fc0b4ceee521ff278e8a8e8",
+    "data/tinysentences/parse_summaries.py": "2317159364dbbe7206f252d2939cab5d375395e0a536be0a1f60c5df88b31702",
+    "data/tinysentences/prepare.py": "35263b287f93337fda4207ed1991e9e8a820ab75139411bb7dd5f0a8a25bcbd9",
+    "data/tinysentences/sentences_only.py": "5e4d72bf612df27dad46cf606a40402606d897043ae24274dc2dfa88ec0b3e79",
+    "data/tinysentences/stats.py": "2665e9e6fac4a8f6c3e5bd8b07cbfd941ba6359a43981d89bdef58c136fdd3a9",
+    "data/tinysentences/summarize/merge_summaries_char.py": "bce94f4d9a824c2d25e45448023d59b7fbc42aba22b33efbc067cdbba93e1934",
+    "data/tinysentences/summarize/merge_summaries_word.py": "f6079110332cc38b2dd3571c30bedcca97098c39498f45eec1ef6e60cce7ac81",
+    "data/tinysentences/summarize/summarize_gemini.py": "282a438efce0dee351fd9833a59c56e4480bed4314dae40844201794ada3bd96",
+    "data/tinysentences/summarize/summarize_gpt.py": "fc08cbc37d6161e5b0057a1ede43272d96f7274524917d6c4d42c22f505c6e09",
+    "data/tinysentences/summarize/summarize_llama.py": "7f3f8230652a5b88ea2856b273d497cd5b633580b430c909e4f996cd0232832d",
+    "data/tinysentences/summarize/summarize_qwen.py": "b07e3c226ab0fdf5b21b025263adfa7c2625d640f0fac3c787bc4a2c9e32107f",
+    "data/tinysentences/summarize/summarize_together.py": "8c26ea052f2a64c2653d39f1ea5c13983dfbc9547f21cd7d7ea9d9b39033bffe",
+    "data/tinysentences/test.py": "9749244da77c078f550a78751664b45bb92067682f3aa27c2338419f1186c844",
+    "data/tinysentences/test_regex.py": "46652e189a0d5f5c0c7a2c90b857082004bf6727ef1d3ad42b631ac7bb0aa88f",
+    "data/tinysentences/tinysentences_char_dataset.py": "152ab9c30aec669a9db14df9e90412597870105eff9cfb737b876aad2323a206",
+    "data/tinysentences/tinysentences_dataset.py": "f90251d21871f6a7ad5ebd2ad0a6e59cd0b02ca245f7e46f93e6d970bf657bc7",
+    "data/tinysentences/tinysentences_vae.py": "30f7adde5413348faf5066448706747fe26a7fedd510ffea720aa0bf8283f6f8",
+    "data/tinysentences/to_csv.py": "c06c74648ed95b4494835c631c94fe1a00a9ff47a272f5e38546a7f94ff9be8b",
+    "data/tinysentences/tokenize/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinysentences/tokenize/common.py": "e877c74df2a48d15991492b126cd0c5fe26731d61f3aa36d83ebf2cb9bda34a0",
+    "data/tinysentences/tokenize/tokenizer.py": "c16abc50b92a163f6978542b36b03c9a14aa00153c73117c4d52084d5de25c2a",
+    "data/tinysentences/tokenize/train_bpe.py": "fa8e55978dd5ca90117200648b468fb22822bf6981f15f0394c815e72eae1bdb",
+    "data/tinystories/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "data/tinystories/download.py": "5afce37e912f6965b6722f0bc028076a655fea78304f1a213e4862dbe223cdcb",
+    "data/tinystories/stats.py": "216650fe19b78e8431c4512ff87a5c84af64d9e56ab8fdaf3952ca035725ea1b",
+    "data/wikitext/parse_wikitext.py": "ab583e0328f82afc3ba242410e8076be42ef1a16a51d66f3eb46a713cca591d0",
+    "evaluation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "evaluation/eval_random_gen_ppl_checkpoint.py": "edf98c0975f54af690fff4b5448f7cbf1c5b95fae5e85421926a663a26dc4c0a",
+    "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+    "evaluation/gen_ppl_ncp.py": "96d68adee372e5186d0470659171eb03829e3d37f611e8f5917b45fb61e3a4ad",
+    "evaluation/gen_ppl_ntp.py": "b2638622bc041fa88e9d538be4229b3723ebec19c5b99fd6e5b73db4776dc952",
+    "evaluation/guess.py": "c27803225dd21b2deff1e7e494b9e3666180b93776340164ce6e953dacc50656",
+    "evaluation/ppl.py": "f3b581cb4849b0b2aa518729b8338838c7055697c3a26028dedb819c335ac838",
+    "evaluation/run_genppl_multiseed.py": "f8c9ac982f4caa4335b5f4d1a22d70f56dc11a706106a3dfb31d344eb8c93c2e",
+    "evaluation/run_genppl_release_ts.py": "d1bc5c00089118d9cf6c05c77459329ad40c2f8bc347b5aeedda8f5318de8796",
+    "evaluation/run_genppl_sweep.py": "ade4f39e373125848fdd32938e8d7bf86a0ec5108cca9bd75f0a966d092a798e",
+    "evaluation/show_8k_samples.py": "30ad825d5a24375e178750dfaaac45bc486bc3587f2e7d6d305d53bf37f8d226",
+    "evaluation/show_low_sample.py": "d696f1e54ca67e87565dce5864e0e6e2ad4194108cfb1c9c1e06eebd97008da6",
+    "evaluation/summarize_gen_ppl.py": "307f3d0d85884879295d5268aaf7f399f654b1b330aaa5728d6824ad24c6d15a",
+    "evaluation/sweep_gen_ppl_checkpoints.py": "3a9141e27d84b49c5493194dfc5a70bfa0a915cdf5ceb7a46afcf27cc6f33acb",
+    "evaluation/sweep_gen_ppl_ncp.py": "d91c2959cfed51d224ccb1c0d0f2c3b9a35c4027cc9ab8c65aa7603d587de761",
+    "evaluation/sweep_gen_ppl_ncp_scaling.py": "05249c1e00c5eb05042b6ea5b1ca008ecb8f0e4a77537ee755a27c153c25a17c",
+    "evaluation/sweep_gen_ppl_ncp_yolo.py": "4b26ef25b27af2f7fcbcb5ff3c312d47d2a4746ba759155d28802e978a681755",
+    "evaluation/sweep_gen_ppl_ntp.py": "14a04074741ee44580ec6557870d42534dd172d26c5be3e943ed5baf1b727303",
+    "evaluation/sweep_random_llama_yolo.py": "4062b54289fcd9407d15feb1ffd241b8722fd717c700350c22472fecec5ae46c",
+    "evaluation/val_oracle_ppl.py": "1a23f3c2897b6a6a3d239ed533979de35352d303bae094f4006bef4156f935c0",
+    "evaluation/val_vqvae_ppl.py": "4c293c502d3faf982d4ad66566b7a037faa7d28191d929de1621cb67f5ae8b5f",
+    "models/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "models/masked_lm.py": "80279614d74efc7b4c25306789d74e2f71045c54e4e4f485ee365f664705d7f6",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/next_token.py": "d0d292b9f5c032a351cbca1a8f0a6d41c3062890cb3e3f5b67d4e23a7cffeb77",
+    "models/quant.py": "b7b73abd4e55e10454cb680dea82124c38c222d37236c4c76b5c7d21123ac1b1",
+    "models/semantic_input.py": "b06658d4bc641397cbb973255eadfd2d6087fde244d801d3c70d29e3ce841bf2",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "scripts/benchmark_iclr_debug.py": "dba86e8d578eec2188c633a479ccf673d58f3e5580b9074869cae1a352b8a043",
+    "scripts/benchmark_mstok_compile.py": "aad909da6f05ba25ac3d8f6b3cf450103bed55d0b93b1490764c0ef54f35b606",
+    "scripts/eval_owt_ckpts_gpt2_large.py": "4977a2d614698f60032fb6fd7fef614eb503d5b3883f499146a4fc048214a509",
+    "scripts/evaluate_mstok_sidecar.py": "9ba33e865fa0fad34bd2ab375d7fe38f37c441e2d96348f0e389e29dd9643455",
+    "scripts/evaluations/sweep_gen_ppl_shared_vocab_ncp.py": "2311ac71f9beba282426faa21390a5898c7d183a95f529755d6e1d6e8fc24050",
+    "scripts/neurips/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "scripts/neurips/eval_gen_ppl_ncp_owt.py": "b92eff1ce198eb4ac8c88da7113b41d095c12acad92db541685175993db3535c",
+    "scripts/neurips/sweep_gen_ppl_owt_gpt2.py": "02adb1629cfa40ff51a75152e4da9c9ee00ead42b7ecda9e78cf7101e6826a67",
+    "scripts/preflight_owtsmall_mstok_ctx256.py": "a4f80921435b6cd136d089ec39bfa5442359d0dffbeefaaa1019178b8364657a",
+    "scripts/prepare_full_owt.py": "b1cb6b74397625c2b0643c4ee744ca3c89dfa6bbb7eeb7e9681e3a36f91a2c24",
+    "scripts/r1_lab/diagnose_at_40k.py": "b1582dcc29ccebc9255eaa77a7d56a7dbdef589ec5d5a9b0af150dede98874ab",
+    "scripts/r1_lab/inspect_summaries.py": "4154e8a6c1b3f68df5c623a57a1a7632b594c46c5f6d1d85cb08c476d6aa9cea",
+    "scripts/r1_lab/overnight_pipeline.py": "53f4e03ce1a5fb43fbca3ae455288f45ae8350b0a16f3d5318763011de6a8f96",
+    "scripts/r1_lab/prime_rope_qknorm_score_cache.py": "a791cdec03b415bfccdfd8d89c7e659aff8a2a5696d16ab7b4350ff608d0391b",
+    "scripts/r1_lab/sweep_gen_ppl_r1lab.py": "c7c0acf9aa6f07bb0a985cf0d5e5692c696e658ebb0470b7c92d9332565f9c06",
+    "scripts/r1_lab/sweep_gen_ppl_r1lab_v15.py": "524aa8680aa1b9fea2689509848c19e5a74a3ad72e78b0388bbca7acd43be22e",
+    "scripts/r1_lab/sweep_gen_ppl_rope_qknorm.py": "7822df057355b0a3a71ea5145742107c6e30f906859096237ff61d765fe36d50",
+    "scripts/r1_lab/train_variant.py": "9d0a2091784609ac9728226afd82f56fd4c1ac2dc08e89e1b17b60d34bb3d55e",
+    "scripts/run_alignment.py": "6312cd9d0976899763589fdcbd258c7681f6e9e96d7396cf35ff022d4695b862",
+    "scripts/run_iclr_debug.py": "440fd11c54e1f0c30c01b2555fc4109e98b296fe564ebabb82510ae3b64927a2",
+    "scripts/run_mstok_semantic.py": "8a7a38a50dd6694950bfa0e7b79718ec653c7e70be99fe13cbf2ee05c28f46a0",
+    "scripts/run_mstok_w1_pilot.py": "35224e462d0d11eb3e4326b0daa8bad7c99e1e30cdc6383310f8e2c8b5bb708f",
+    "scripts/run_substitution.py": "2cd22eaa58dab8b5231984eee340a3a2bdd0b88cc83333f7133447471e7f73f1",
+    "scripts/smoke_test_masked_l0.py": "d24f1270b3219a31212b68bcbb786f649c7c72f96e53f602d87ad6c17f3f73e5",
+    "scripts/test0_diagnostics.py": "ed389c666beef015ccac004f314a5a1428def303b46ac0e442f0236337344549",
+    "scripts/test0_r1_similarity.py": "4a486878bebd418a3b5454ddbf02585726ccbacc2700fdf591aa524766f9c9c9",
+    "scripts/test0_r1_uniqueness.py": "3ee59d4397d5b268151dbb8ca779c7f9bf8461d129d97fcd1a517c48ec2a69a1",
+    "scripts/test0_residual_norms.py": "3c4f68e5960e84796dc0bca4fd4583bf3b598552654f9b565fd5fcabd420f9e1",
+    "scripts/test_gen_equiv.py": "ae0e9c513422ee58bdc050ed6218ee54fc555dfb15ea8968c9ffc62a7a7ea99a",
+    "scripts/training_budget.py": "bb94c19c6dfbdd542946b10476b8d6958b8a189abd141cf62288fdb5fa9a2d38",
+    "scripts/vqvae_utilization.py": "5f90327f1507a5042b2f77b119fe4fc0df4e2131e5bbc21069dffd1fadc059c5",
+    "scripts/xsum/codec_ceiling.py": "ddb751e9152d5c88966e588d89cdceee4c5d91c4139aaedf369f65a45ecf2276",
+    "scripts/xsum/eval_merge.py": "6e313d6ed25227d0418b7422d58e09ceda4fc6dfae7d0fd663417b306241135c",
+    "scripts/xsum/eval_q0.py": "82fe04dccce28d7fc9c6ec8b4c9c91404a49527f9437913f44f51a6853ba9abc",
+    "scripts/xsum/eval_shard.py": "07192f6ba6219da1e8123ceeb99748c86da85322859ed5fa25cebcc3816f7ea8",
+    "scripts/xsum/eval_shard_cfg.py": "95ac4a8e2e3a73a21fa4303c62df80c086f23bcfe5ab34eb07ae90b585052ecb",
+    "scripts/xsum/eval_test.py": "7f92d578c31893e788e5fa7d9eef3098ba5602b3642873ce2ce031342c8be868",
+    "scripts/xsum/eval_uncond_rep.py": "ba4740b29bc091d866e2dfd91881e9a7d5c46a1bfa993d13eb63c0467397ec28",
+    "scripts/xsum/make_warmstart.py": "78793d6ef11fbf658c2837ef12cb843fc4817add8ba73c822b5e408901de3318",
+    "scripts/xsum/mbr_fast.py": "3db6e21d2dcce2e3bcf6f22474535d7eba0232b071b4271d2b172f787aed5c85",
+    "scripts/xsum/mbr_select.py": "107a061014c62395b7f5fd986fba9e55dd738054e2474022005cb7e9e0c9b8fb",
+    "scripts/xsum/mbr_shard_cfg.py": "3e7162375028c391276149eff288e9de40c874a3108ac53de0248198f4d08791",
+    "scripts/xsum/pool_append.py": "e50252f48c4d8d905be0de3eb56ca1d720f8f8545c6fdcb58ed27f0360c4e2ad",
+    "scripts/xsum/prep_xsum_le48.py": "3688ed1abc49d18f75d34f8382ff68a5afbd753b36cefdc0652d0bffb6a5f347",
+    "scripts/xsum/prep_xsum_mix30.py": "4bd47e52637c4b6bca5fd4956c1787b506a7f51b8a5bb0b68f19f728c4d48afb",
+    "scripts/xsum/prep_xsum_mix_split.py": "ec2fe74f08068ae8fda4ed59a86537a1d3f48276d205099bbfe16f8428ca36d5",
+    "scripts/xsum/prep_xsum_pass_uncond.py": "82ea50248d811e5182e63d0b4dfd6f995dd2f47eb963b5bb1c5d09bc533db7c1",
+    "scripts/xsum/prep_xsum_test_all.py": "7389fbdf5612ba524801c5b04109f810b139152ca50101b60acbb9b1e767b256",
+    "scripts/xsum/score_checkpoints.py": "41b412c06cd92cbd4caa6b586da7ddc7e913607a47c1a3aac0ee0a36a20a36be",
+    "tokenization/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "tokenization/bpe_tokenizer.py": "0c8af84868a6aa40f981768f8ab49d28e501d55cce444edb1ad315de53c3445a",
+    "tokenization/build_word_vocab.py": "778f9666f4e7c642dc471773ee81c92c26975db37de1ebfffb605cae211351b5",
+    "tokenization/byte_tokenizer.py": "eb567a1772ae69738884ed9b94330aa39387c49e54d76e4b9752e67674fad810",
+    "tokenization/compression_ratio.py": "ed07ab1cc38d6176362631ce655bc974dd3623b87693f51279d5047e6bc6890f",
+    "tokenization/hf_tokenizer.py": "2ca8fae097f9c640bcb52b4788c37de8e0d9bb2fa971e30a5abb39f90ff97d4c",
+    "tokenization/tokenize.py": "f8ab54338613b6bd84513fdda85be173fd32274c35d97ef39a5c17d584f8108f",
+    "tokenization/tokenize_doc_level.py": "a06c81cf088fb81d2317ed1c3a1417a0e48ae0ea04962824f13f5d3beddc8902",
+    "tokenization/tokenize_vqvae.py": "14730eec3da87f6528e5fbd1166d5da79e10ac1700f7cd0be73bc40716bdebfc",
+    "tokenization/tokenizer.py": "e527d47e4ed8634c673d48a872a07c9cc8828d30988331de2d4bd3b7c21ddcb7",
+    "tokenization/train_bpe.py": "bf4e2a29eeba874024ed95f32fe720ae2320f72cb978361692db73e71be98683",
+    "tokenization/vqvae_tokenizer.py": "d7fa9d595865aa3e0b0d185bfd04721eb297fe5f657c4fca4ce69fc96aca308f",
+    "tokenization/word_tokenizer.py": "a405621bf35ed6cbb29abadd9d1d651e0783e6801fc7645b35e799d5de381ac2",
+    "train_mstok.py": "077b7c99c12daad2bbd986918d0f487411779de05a472af71124098da399b79d",
+    "trainer/alignment_trainer.py": "54f8ee7cca4d407223f2ce5146e6fc561ad748e20f2c702beb12ae040cd290c4",
+    "trainer/corruption.py": "f0f24c94652e6e9cf8e4ef30ae1a302e3232d64e765c931696d454db96e8b57d",
+    "trainer/joint_trainer.py": "8c8ebb55d21b6e223be1f7595c51f1b9257ccac40f8cff28260c1088656e2e49",
+    "trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
+    "trainer/ncp_trainer.py": "73fc23f78f403dc39800b924dfb18ad42ffeab847fed30e931183c3d3acaee68",
+    "trainer/ntp_trainer.py": "5076d7aa29fac70869e1e309a15af6c4d3a91d6de8e00ac113a586841a9866e1",
+    "trainer/onpolicy.py": "6ff1fe4103d71001afe2d73e574d20c7242381e420929db14020141464ec6821",
+    "trainer/plain_two_stage.py": "4c5973ed87100b1b478aa1df0bb16cd3d1eb5844dffe5894003dd67c463f6c9a",
+    "trainer/scheduled_sampling.py": "f53515e23c2cec1231265f21a341ff77d11fa8a6129e0e382d540cc85571f5ed",
+    "trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
+    "trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
+    "trainer/substitution_trainer.py": "33c7833b2e8dfabbb9ee56b258708055d3194b8d7453a1b8d93f59b44ff91465",
+    "utils/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "utils/arg_utils.py": "e9eb7971f6273e9630b56fd87a03725b29183a8c80abb014c4c84f8b1bdb6445",
+    "utils/benchmark.py": "ccf436ef72a4e626c7f671d8e63a683057bf502a3852e5a7142410d1dc3e5501",
+    "utils/data.py": "78cce11e777813755cd171ccaa3247f89ab99ac4079e0157e55c72ce8a5a7fe5",
+    "utils/dist.py": "b2849d3ce03a8b1737e3dd61216e76e4a9e82d63ad1ea74cddf72e2285eaca64",
+    "utils/experiment_config.py": "616a70fefaac99ab0d856a1b6c36787a28fb9af8a4c55fc6d4765012fbaaeb38",
+    "utils/iclr_training.py": "10c6857fa2319952fe0df18067730b42e3ff755244e2b059f6c95882f8d27c8e",
+    "utils/logging.py": "901e67515cc65a35f575f05d16a3b9ef4a5b2cb0ea252cd7abbc80ecce9e88e8",
+    "utils/lr.py": "4d1ea2c25340afc4bdde8b582dab36f174e0b2965fa7bafa2a7539019dbe2b7e",
+    "utils/misc.py": "be4a323116a0956a77960d1baf613f268057d6a36023663d2af9f8e4728cbe77",
+    "utils/plot_levels.py": "916b39885ced191960c26c334a3b16f9c99751f6c29c46bd69d14fee414965f3",
+    "utils/registry.py": "f7a44c6d18433100873d5df462b9f08a8bdeb11cf3d92226334c7e5ece318318",
+    "utils/train.py": "43eef988000f3a5e20856653d725337a0fe0dbe276f0cf80e1b200292abe812d",
+    "train_iclr_debug.py": "e0a0adf4a30ef55925f9017de24d434299749332e58a507a6424265f7a5f796e",
+    "config/iclr-debug/README.md": "1c757d2ff37026d08c22324ec44cfa760b0599af62c8ad9b728ada471a8f2905",
+    "config/iclr-debug/benchmark-16sq.yaml": "18c9cef7032f9d3806d8b0c8b20fe0fc7d1ea98ad1d2440e24a67ee873c97f9b",
+    "config/iclr-debug/full-owt-audit.json": "fefa394834f6e9f65944d3b514633c29061071ebe6ca60fce21a6edef9748c1e",
+    "config/iclr-debug/training-defaults.yaml": "3b144b8d53f96fe035e36edca96aac456e6cf21202373a467a3cd875fe86b474",
+    "config/iclr-debug/NCM-BUDGET-AUDIT.md": "e734caeab7edb612de7ff508a8c0ec52445530ab7a35895b5b793d6a7b752e4b",
+    "config/iclr-debug/TRAINING.md": "91555457c5cf6af078075673ce96cb440bc8e22aeae5211c252d94f92ea87ac7",
+    "config/iclr-debug/BENCHMARK.md": "2781ea506212e12a0640a4b0d2a2c881355ec85756ca4014bccb1b1aedccdf59",
+    "config/iclr-debug/HF-MODEL-CARD.md": "de1fe176ebde262eacdf32af7d3f8c6497a764dd98e5a9094dad26de4aade38e"
+  },
+  "dataset": {
+    "dataset": "hazyresearch/ncm-tokenized-datasets",
+    "revision": "951e04517fd53d57521d8e09abd6511a662787d0",
+    "dtype": "little-endian uint16",
+    "eos_id": 50256,
+    "offset_units": "token positions; [0, each EOS position + 1] with final sentinel",
+    "sampling": "length-weighted within-segment windows; pad short segments with 50257",
+    "caveat": "EOS segments are not independently verified source-document identities",
+    "splits": {
+      "train": {
+        "bytes": 18070847356,
+        "tokens": 9035423678,
+        "sha256": "b24b78a07f56974fd0c9855dc16f25b836d2cb172be23eb5d3c650cd6d73f56a",
+        "min_token_id": 0,
+        "max_token_id": 50256,
+        "segments": 8009770,
+        "trailing_tokens": 0,
+        "eos_only_segments": 0,
+        "length_percentiles": {
+          "min": 132.0,
+          "p25": 416.0,
+          "p50": 718.0,
+          "p75": 1249.0,
+          "p99": 7610.0,
+          "max": 131288.0
+        },
+        "segments_shorter_than_256": 673011,
+        "expected_padding_fraction_at_256": 1.5955357567300495e-05,
+        "offsets_status": "created",
+        "offsets_sha256": "7e84564946b8cf8190dbdc1e86bdf2fe8cfd9235a517328c80b4df038ea0028b",
+        "offset_entries": 8009771
+      },
+      "valid": {
+        "bytes": 9186834,
+        "tokens": 4593417,
+        "sha256": "1f2ee17e08327a39d4e2417948d02c7b998d626d2dcb2661197a6e208f58266f",
+        "min_token_id": 0,
+        "max_token_id": 50256,
+        "segments": 3999,
+        "trailing_tokens": 0,
+        "eos_only_segments": 0,
+        "length_percentiles": {
+          "min": 143.0,
+          "p25": 433.0,
+          "p50": 734.0,
+          "p75": 1292.5,
+          "p99": 7122.719999999997,
+          "max": 24868.0
+        },
+        "segments_shorter_than_256": 363,
+        "expected_padding_fraction_at_256": 1.6064002171981108e-05,
+        "offsets_status": "created",
+        "offsets_sha256": "09a748ce1dcec7de36c77c48e39ccbf5c0293cc073689fa7c5f48bdfb9bb9445",
+        "offset_entries": 4000
+      }
+    },
+    "overlap": {
+      "exact_train_segment_matches": 0,
+      "unique_validation_segments_in_train": 0,
+      "unique_validation_segments": 3999,
+      "near_duplicates_checked": false
+    }
+  },
+  "hf_repo": "iskhare/iclr-debug",
+  "generation_evaluation": true,
+  "evaluation_reference_dtype": "float64",
+  "external_tokenizer": {
+    "path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt",
+    "step": 128747,
+    "sha256": "0078f567fdf3b261aba666f8d591bedc32403b1c626eed9ef05c4f5c23a2cedd"
+  },
+  "versions": {
+    "torch": "2.13.0+cu130",
+    "transformers": "4.44.2",
+    "numpy": "1.26.4",
+    "hydra-core": "1.3.6"
+  }
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/progress.json

@@ -0,0 +1,14 @@
+{
+  "step": 39000,
+  "checkpoint": "/home/ubuntu/mstok-results/iclr-debug-generator-decay27b-20260917/generator/checkpoint-iter-39000.pt",
+  "nonpadding_tokens": 40893810223,
+  "validation": {
+    "loss": 5.625601601600647,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.625601601600647,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.80978260869565
+  }
+}

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/training-to-954.log

@@ -0,0 +1,185 @@
+wandb: Currently logged in as: iskhare (mstok). Use `wandb login --relogin` to force relogin
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+wandb: Tracking run with wandb version 0.17.9
+wandb: Run data is saved locally in /home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/wandb/run-20260917_222739-iclr-debug-generator-decay27b-20260917-generator
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run iclr-debug-generator-decay27b-20260917-generator
+wandb: ⭐️ View project at https://wandb.ai/mstok/iclr-debug
+wandb: 🚀 View run at https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-generator-decay27b-20260917-generator
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+VQVAE, Total parameter count: 223.75M
+VQVAE, Non embedding parameter count: 204.34M
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+NCP Transformer, Total parameter count: 104.61M
+NCP Transformer, Non embedding parameter count: 104.30M
+/home/ubuntu/mstok-runs/iclr-debug-generator-decay27b-20260917/train_iclr_debug.py:129: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
+  ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
+{"step": 1, "stop_at": 954, "target": 128747, "training/loss": 9.871083557605743, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.871083557605743, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.004180602006688963, "training/generator_lr": 4.444444444444444e-06, "training/grad_norm": 0.14711065590381622, "training/positions": 1048576, "training/nonpadding_tokens": 1048576, "throughput/steps_per_second": 0.040056736496171794, "throughput/raw_tokens_per_second": 42002.532528209835, "throughput/peak_reserved_gib": 23.4921875}
+{"step": 10, "stop_at": 954, "target": 128747, "training/loss": 9.844660883148512, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.844660883148512, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.006423320791527313, "training/generator_lr": 2.4444444444444445e-05, "training/grad_norm": 0.11732996255159378, "training/positions": 10485760, "training/nonpadding_tokens": 10485517, "throughput/steps_per_second": 0.4089769276301097, "throughput/raw_tokens_per_second": 428832.34848962387, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 20, "stop_at": 954, "target": 128747, "training/loss": 9.761483243852854, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.761483243852854, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.10068392035953178, "training/generator_lr": 4.666666666666667e-05, "training/grad_norm": 0.07661070674657822, "training/positions": 20971520, "training/nonpadding_tokens": 20971118, "throughput/steps_per_second": 0.40854297967511893, "throughput/raw_tokens_per_second": 428381.8676224407, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 30, "stop_at": 954, "target": 128747, "training/loss": 9.691915126144886, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.691915126144886, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.12911201400501673, "training/generator_lr": 6.88888888888889e-05, "training/grad_norm": 0.06473804265260696, "training/positions": 31457280, "training/nonpadding_tokens": 31456759, "throughput/steps_per_second": 0.40819095946349926, "throughput/raw_tokens_per_second": 428014.38603798056, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 40, "stop_at": 954, "target": 128747, "training/loss": 9.617637950181962, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.617637950181962, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.14391722408026755, "training/generator_lr": 9.111111111111112e-05, "training/grad_norm": 0.05417126789689064, "training/positions": 41943040, "training/nonpadding_tokens": 41942305, "throughput/steps_per_second": 0.4082063528359705, "throughput/raw_tokens_per_second": 428026.6490153799, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 50, "stop_at": 954, "target": 128747, "training/loss": 9.540264104306697, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.540264104306697, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.1579304060409699, "training/generator_lr": 0.00011333333333333333, "training/grad_norm": 0.046573422849178314, "training/positions": 52428800, "training/nonpadding_tokens": 52427940, "throughput/steps_per_second": 0.40827410034268924, "throughput/raw_tokens_per_second": 428101.3196146814, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 60, "stop_at": 954, "target": 128747, "training/loss": 9.472410047054291, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.472410047054291, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.21007361778846154, "training/generator_lr": 0.00013555555555555556, "training/grad_norm": 0.040664225816726685, "training/positions": 62914560, "training/nonpadding_tokens": 62913625, "throughput/steps_per_second": 0.4082463355870231, "throughput/raw_tokens_per_second": 428074.2477369815, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 70, "stop_at": 954, "target": 128747, "training/loss": 9.417964617908002, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.417964617908002, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.2791727633779264, "training/generator_lr": 0.00015777777777777776, "training/grad_norm": 0.04755992069840431, "training/positions": 73400320, "training/nonpadding_tokens": 73399129, "throughput/steps_per_second": 0.40795361411389064, "throughput/raw_tokens_per_second": 427759.92526056565, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 80, "stop_at": 954, "target": 128747, "training/loss": 9.377101661264897, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.377101661264897, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.2935272522993311, "training/generator_lr": 0.00017999999999999998, "training/grad_norm": 0.03830907493829727, "training/positions": 83886080, "training/nonpadding_tokens": 83884677, "throughput/steps_per_second": 0.40793310448662184, "throughput/raw_tokens_per_second": 427740.2147883489, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 90, "stop_at": 954, "target": 128747, "training/loss": 9.345998856425286, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.345998856425286, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.4252586747491639, "training/generator_lr": 0.00020222222222222223, "training/grad_norm": 0.06383036077022552, "training/positions": 94371840, "training/nonpadding_tokens": 94370271, "throughput/steps_per_second": 0.40765827425937967, "throughput/raw_tokens_per_second": 427453.9154624506, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 100, "stop_at": 954, "target": 128747, "training/loss": 9.3251358166337, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.3251358166337, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.5700887724707357, "training/generator_lr": 0.00022444444444444446, "training/grad_norm": 0.10724775493144989, "training/positions": 104857600, "training/nonpadding_tokens": 104855825, "throughput/steps_per_second": 0.40800465790836943, "throughput/raw_tokens_per_second": 427815.4872749735, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 110, "stop_at": 954, "target": 128747, "training/loss": 9.30225510597229, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.30225510597229, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.7181506715091973, "training/generator_lr": 0.0002466666666666667, "training/grad_norm": 0.05290738120675087, "training/positions": 115343360, "training/nonpadding_tokens": 115341579, "throughput/steps_per_second": 0.40772746738335813, "throughput/raw_tokens_per_second": 427532.99220249173, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 120, "stop_at": 954, "target": 128747, "training/loss": 9.274820706248283, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.274820706248283, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.8931398933946488, "training/generator_lr": 0.00026888888888888893, "training/grad_norm": 0.05407208576798439, "training/positions": 125829120, "training/nonpadding_tokens": 125827327, "throughput/steps_per_second": 0.4078665861646931, "throughput/raw_tokens_per_second": 427678.62401432585, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 130, "stop_at": 954, "target": 128747, "training/loss": 9.250935723632574, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.250935723632574, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.034989679138796, "training/generator_lr": 0.00029111111111111113, "training/grad_norm": 0.09266811609268188, "training/positions": 136314880, "training/nonpadding_tokens": 136312889, "throughput/steps_per_second": 0.4080129739936707, "throughput/raw_tokens_per_second": 427824.5335615022, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 140, "stop_at": 954, "target": 128747, "training/loss": 9.215728542208671, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.215728542208671, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.2045473845108696, "training/generator_lr": 0.0003133333333333334, "training/grad_norm": 0.05760117992758751, "training/positions": 146800640, "training/nonpadding_tokens": 146798407, "throughput/steps_per_second": 0.4081664461218752, "throughput/raw_tokens_per_second": 427983.66178069526, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 150, "stop_at": 954, "target": 128747, "training/loss": 9.179465129226447, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.179465129226447, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.397830006270903, "training/generator_lr": 0.0003355555555555556, "training/grad_norm": 0.14713037014007568, "training/positions": 157286400, "training/nonpadding_tokens": 157284143, "throughput/steps_per_second": 0.40807877914102925, "throughput/raw_tokens_per_second": 427900.63452751393, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 160, "stop_at": 954, "target": 128747, "training/loss": 9.154566588997842, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.154566588997842, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.463449127299331, "training/generator_lr": 0.00035777777777777777, "training/grad_norm": 0.07448174804449081, "training/positions": 167772160, "training/nonpadding_tokens": 167769767, "throughput/steps_per_second": 0.40834239634819103, "throughput/raw_tokens_per_second": 428172.4831366104, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 170, "stop_at": 954, "target": 128747, "training/loss": 9.113213916867972, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.113213916867972, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.7127779447115385, "training/generator_lr": 0.00038, "training/grad_norm": 0.0707898736000061, "training/positions": 178257920, "training/nonpadding_tokens": 178255314, "throughput/steps_per_second": 0.408420565060771, "throughput/raw_tokens_per_second": 428251.3030711272, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 180, "stop_at": 954, "target": 128747, "training/loss": 9.085574293136597, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.085574293136597, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.8674700172449832, "training/generator_lr": 0.0004022222222222222, "training/grad_norm": 0.12432141602039337, "training/positions": 188743680, "training/nonpadding_tokens": 188740985, "throughput/steps_per_second": 0.4083875000309074, "throughput/raw_tokens_per_second": 428221.69658365846, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 190, "stop_at": 954, "target": 128747, "training/loss": 9.05402339398861, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.05402339398861, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.0216803407190636, "training/generator_lr": 0.00042444444444444447, "training/grad_norm": 0.07192572206258774, "training/positions": 199229440, "training/nonpadding_tokens": 199226665, "throughput/steps_per_second": 0.40873834507605844, "throughput/raw_tokens_per_second": 428589.94901971245, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 200, "stop_at": 954, "target": 128747, "training/loss": 9.018498958647251, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.018498958647251, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.2147180053302677, "training/generator_lr": 0.00044666666666666666, "training/grad_norm": 0.11893639713525772, "training/positions": 209715200, "training/nonpadding_tokens": 209712037, "throughput/steps_per_second": 0.40857115212668466, "throughput/raw_tokens_per_second": 428402.051851688, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 210, "stop_at": 954, "target": 128747, "training/loss": 8.987547859549522, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.987547859549522, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.3577713471989967, "training/generator_lr": 0.0004688888888888889, "training/grad_norm": 0.13652347028255463, "training/positions": 220200960, "training/nonpadding_tokens": 220197684, "throughput/steps_per_second": 0.40863238412289826, "throughput/raw_tokens_per_second": 428477.4932681116, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 220, "stop_at": 954, "target": 128747, "training/loss": 8.9624773144722, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.9624773144722, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.496359936768395, "training/generator_lr": 0.0004911111111111111, "training/grad_norm": 0.12405138462781906, "training/positions": 230686720, "training/nonpadding_tokens": 230683308, "throughput/steps_per_second": 0.4087982310250671, "throughput/raw_tokens_per_second": 428650.4542393988, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 230, "stop_at": 954, "target": 128747, "training/loss": 8.936211570352317, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.936211570352317, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.5985315635451505, "training/generator_lr": 0.0004999999536078684, "training/grad_norm": 0.10296963155269623, "training/positions": 241172480, "training/nonpadding_tokens": 241168894, "throughput/steps_per_second": 0.40878647647415084, "throughput/raw_tokens_per_second": 428636.57547066855, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 240, "stop_at": 954, "target": 128747, "training/loss": 8.90674582272768, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.90674582272768, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.755632381375418, "training/generator_lr": 0.0004999995824709212, "training/grad_norm": 0.1342928558588028, "training/positions": 251658240, "training/nonpadding_tokens": 251654582, "throughput/steps_per_second": 0.4090183719025809, "throughput/raw_tokens_per_second": 428883.903403843, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 250, "stop_at": 954, "target": 128747, "training/loss": 8.880627527832985, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.880627527832985, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.8724818405100336, "training/generator_lr": 0.0004999988401975889, "training/grad_norm": 0.12204146385192871, "training/positions": 262144000, "training/nonpadding_tokens": 262140041, "throughput/steps_per_second": 0.4091249482977521, "throughput/raw_tokens_per_second": 428986.28712531994, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 260, "stop_at": 954, "target": 128747, "training/loss": 8.851174211502075, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.851174211502075, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.0409274404264215, "training/generator_lr": 0.000499997726788996, "training/grad_norm": 0.09389498829841614, "training/positions": 272629760, "training/nonpadding_tokens": 272625566, "throughput/steps_per_second": 0.4091722457187704, "throughput/raw_tokens_per_second": 429038.581179031, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 270, "stop_at": 954, "target": 128747, "training/loss": 8.834697999805211, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.834697999805211, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.115928420255017, "training/generator_lr": 0.0004999962422468292, "training/grad_norm": 0.2540742754936218, "training/positions": 283115520, "training/nonpadding_tokens": 283111224, "throughput/steps_per_second": 0.40914011252446153, "throughput/raw_tokens_per_second": 429010.32940130203, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 280, "stop_at": 954, "target": 128747, "training/loss": 8.82238697335124, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.82238697335124, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.1824179556856187, "training/generator_lr": 0.0004999943865733373, "training/grad_norm": 0.07487369328737259, "training/positions": 293601280, "training/nonpadding_tokens": 293596820, "throughput/steps_per_second": 0.40909369565124826, "throughput/raw_tokens_per_second": 428959.1218745946, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 290, "stop_at": 954, "target": 128747, "training/loss": 8.79404194355011, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.79404194355011, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.3248882995401337, "training/generator_lr": 0.0004999921597713312, "training/grad_norm": 0.19123147428035736, "training/positions": 304087040, "training/nonpadding_tokens": 304082455, "throughput/steps_per_second": 0.40900614031880356, "throughput/raw_tokens_per_second": 428868.9100141758, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 300, "stop_at": 954, "target": 128747, "training/loss": 8.776946759968997, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.776946759968997, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.3914839830685617, "training/generator_lr": 0.0004999895618441844, "training/grad_norm": 0.1234544888138771, "training/positions": 314572800, "training/nonpadding_tokens": 314568065, "throughput/steps_per_second": 0.4090809428768252, "throughput/raw_tokens_per_second": 428946.32254386676, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 310, "stop_at": 954, "target": 128747, "training/loss": 8.756403009593487, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.756403009593487, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.5498912390259196, "training/generator_lr": 0.0004999865927958323, "training/grad_norm": 0.11858051270246506, "training/positions": 325058560, "training/nonpadding_tokens": 325053596, "throughput/steps_per_second": 0.409160627486105, "throughput/raw_tokens_per_second": 429026.6443485006, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 320, "stop_at": 954, "target": 128747, "training/loss": 8.75254106298089, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.75254106298089, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.5737696618938126, "training/generator_lr": 0.0004999832526307726, "training/grad_norm": 0.1067967414855957, "training/positions": 335544320, "training/nonpadding_tokens": 335538862, "throughput/steps_per_second": 0.4091051504153106, "throughput/raw_tokens_per_second": 428957.6324074542, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 330, "stop_at": 954, "target": 128747, "training/loss": 8.727130230516195, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.727130230516195, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.711794850020903, "training/generator_lr": 0.0004999795413540649, "training/grad_norm": 0.2058543860912323, "training/positions": 346030080, "training/nonpadding_tokens": 346024551, "throughput/steps_per_second": 0.409093990535843, "throughput/raw_tokens_per_second": 428963.2356527793, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 340, "stop_at": 954, "target": 128747, "training/loss": 8.718689168989659, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.718689168989659, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.753760908758361, "training/generator_lr": 0.0004999754589713314, "training/grad_norm": 0.1372404843568802, "training/positions": 356515840, "training/nonpadding_tokens": 356510227, "throughput/steps_per_second": 0.40941325148856034, "throughput/raw_tokens_per_second": 429297.4705215561, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 350, "stop_at": 954, "target": 128747, "training/loss": 8.698520158231258, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.698520158231258, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.869996341973244, "training/generator_lr": 0.0004999710054887561, "training/grad_norm": 0.11573535203933716, "training/positions": 367001600, "training/nonpadding_tokens": 366995799, "throughput/steps_per_second": 0.4092907389225626, "throughput/raw_tokens_per_second": 429164.7511905733, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 360, "stop_at": 954, "target": 128747, "training/loss": 8.680578867346048, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.680578867346048, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.9775913200250836, "training/generator_lr": 0.0004999661809130856, "training/grad_norm": 0.1820361167192459, "training/positions": 377487360, "training/nonpadding_tokens": 377481238, "throughput/steps_per_second": 0.4092263709105272, "throughput/raw_tokens_per_second": 429091.8149373708, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 370, "stop_at": 954, "target": 128747, "training/loss": 8.673419808596373, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.673419808596373, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.026470069502508, "training/generator_lr": 0.0004999609852516284, "training/grad_norm": 0.1413300782442093, "training/positions": 387973120, "training/nonpadding_tokens": 387966997, "throughput/steps_per_second": 0.4092763431386803, "throughput/raw_tokens_per_second": 429157.3098553505, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 380, "stop_at": 954, "target": 128747, "training/loss": 8.657889736443758, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.657889736443758, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.1103580946906355, "training/generator_lr": 0.0004999554185122548, "training/grad_norm": 0.16990049183368683, "training/positions": 398458880, "training/nonpadding_tokens": 398452694, "throughput/steps_per_second": 0.40917859331807166, "throughput/raw_tokens_per_second": 429052.2748419524, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 390, "stop_at": 954, "target": 128747, "training/loss": 8.644180211424828, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.644180211424828, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.20180386444398, "training/generator_lr": 0.0004999494807033976, "training/grad_norm": 0.1780761182308197, "training/positions": 408944640, "training/nonpadding_tokens": 408938400, "throughput/steps_per_second": 0.4092390462517666, "throughput/raw_tokens_per_second": 429116.03227164265, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 400, "stop_at": 954, "target": 128747, "training/loss": 8.631474277377128, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.631474277377128, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.283026755852843, "training/generator_lr": 0.000499943171834052, "training/grad_norm": 0.13008002936840057, "training/positions": 419430400, "training/nonpadding_tokens": 419424058, "throughput/steps_per_second": 0.40910664641377315, "throughput/raw_tokens_per_second": 428975.23798217514, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 410, "stop_at": 954, "target": 128747, "training/loss": 8.617905244976281, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.617905244976281, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.374452929034281, "training/generator_lr": 0.0004999364919137746, "training/grad_norm": 0.219180166721344, "training/positions": 429916160, "training/nonpadding_tokens": 429909611, "throughput/steps_per_second": 0.40950230483159894, "throughput/raw_tokens_per_second": 429385.8120933887, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 420, "stop_at": 954, "target": 128747, "training/loss": 8.613953599333763, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.613953599333763, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.40461368624582, "training/generator_lr": 0.0004999294409526847, "training/grad_norm": 0.16120874881744385, "training/positions": 440401920, "training/nonpadding_tokens": 440395082, "throughput/steps_per_second": 0.409645408558859, "throughput/raw_tokens_per_second": 429532.5051727068, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 430, "stop_at": 954, "target": 128747, "training/loss": 8.597871249169112, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.597871249169112, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.495283758361204, "training/generator_lr": 0.0004999220189614631, "training/grad_norm": 0.10934364050626755, "training/positions": 450887680, "training/nonpadding_tokens": 450880746, "throughput/steps_per_second": 0.4095650213174139, "throughput/raw_tokens_per_second": 429456.1199687239, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 440, "stop_at": 954, "target": 128747, "training/loss": 8.581347691267728, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.581347691267728, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.59940851013796, "training/generator_lr": 0.0004999142259513533, "training/grad_norm": 0.1813250631093979, "training/positions": 461373440, "training/nonpadding_tokens": 461366347, "throughput/steps_per_second": 0.4094818471282451, "throughput/raw_tokens_per_second": 429366.32657297736, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 450, "stop_at": 954, "target": 128747, "training/loss": 8.575579819083213, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.575579819083213, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.63100308319398, "training/generator_lr": 0.0004999060619341604, "training/grad_norm": 0.16198433935642242, "training/positions": 471859200, "training/nonpadding_tokens": 471852012, "throughput/steps_per_second": 0.4095541859437562, "throughput/raw_tokens_per_second": 429444.79931539367, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 460, "stop_at": 954, "target": 128747, "training/loss": 8.57096881940961, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.57096881940961, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.657923873850335, "training/generator_lr": 0.0004998975269222515, "training/grad_norm": 0.16562889516353607, "training/positions": 482344960, "training/nonpadding_tokens": 482337515, "throughput/steps_per_second": 0.40968604687135984, "throughput/raw_tokens_per_second": 429576.42735277844, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 470, "stop_at": 954, "target": 128747, "training/loss": 8.556777696311475, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.556777696311475, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.746634288775083, "training/generator_lr": 0.000499888620928556, "training/grad_norm": 0.13155809044837952, "training/positions": 492830720, "training/nonpadding_tokens": 492823060, "throughput/steps_per_second": 0.40966747414455645, "throughput/raw_tokens_per_second": 429558.6735179083, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 480, "stop_at": 954, "target": 128747, "training/loss": 8.548590744286775, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.548590744286775, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.82834056229097, "training/generator_lr": 0.000499879343966565, "training/grad_norm": 0.23985089361667633, "training/positions": 503316480, "training/nonpadding_tokens": 503308713, "throughput/steps_per_second": 0.4096589314023696, "throughput/raw_tokens_per_second": 429554.1403036051, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 490, "stop_at": 954, "target": 128747, "training/loss": 8.545582038909197, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.545582038909197, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.85551284228679, "training/generator_lr": 0.0004998696960503315, "training/grad_norm": 0.11073504388332367, "training/positions": 513802240, "training/nonpadding_tokens": 513794206, "throughput/steps_per_second": 0.4096692793545344, "throughput/raw_tokens_per_second": 429558.4360987015, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 500, "stop_at": 954, "target": 128747, "training/loss": 8.527390617877245, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.527390617877245, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.957205986099498, "training/generator_lr": 0.0004998596771944709, "training/grad_norm": 0.21505460143089294, "training/positions": 524288000, "training/nonpadding_tokens": 524279876, "throughput/steps_per_second": 0.4095943684861191, "throughput/raw_tokens_per_second": 429487.13818038447, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 510, "stop_at": 954, "target": 128747, "training/loss": 8.521949668973685, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.521949668973685, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.9849106396321075, "training/generator_lr": 0.00049984928741416, "training/grad_norm": 0.18125534057617188, "training/positions": 534773760, "training/nonpadding_tokens": 534765323, "throughput/steps_per_second": 0.40977994721772865, "throughput/raw_tokens_per_second": 429672.59182142915, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 520, "stop_at": 954, "target": 128747, "training/loss": 8.51322649717331, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.51322649717331, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.072979266826923, "training/generator_lr": 0.000499838526725138, "training/grad_norm": 0.17534515261650085, "training/positions": 545259520, "training/nonpadding_tokens": 545250857, "throughput/steps_per_second": 0.40969322252261053, "throughput/raw_tokens_per_second": 429585.2214330399, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 530, "stop_at": 954, "target": 128747, "training/loss": 8.507230238616467, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.507230238616467, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.104608133883779, "training/generator_lr": 0.0004998273951437052, "training/grad_norm": 0.19355307519435883, "training/positions": 555745280, "training/nonpadding_tokens": 555736516, "throughput/steps_per_second": 0.4097810784446137, "throughput/raw_tokens_per_second": 429682.46532224695, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 540, "stop_at": 954, "target": 128747, "training/loss": 8.497686830163001, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.497686830163001, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.1918879990593645, "training/generator_lr": 0.0004998158926867246, "training/grad_norm": 0.2563135623931885, "training/positions": 566231040, "training/nonpadding_tokens": 566221693, "throughput/steps_per_second": 0.40968040684812573, "throughput/raw_tokens_per_second": 429557.15792346105, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 550, "stop_at": 954, "target": 128747, "training/loss": 8.490225145965814, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.490225145965814, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.228912455581104, "training/generator_lr": 0.0004998040193716204, "training/grad_norm": 0.1352270543575287, "training/positions": 576716800, "training/nonpadding_tokens": 576707372, "throughput/steps_per_second": 0.4096360303449013, "throughput/raw_tokens_per_second": 429531.1921030894, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 560, "stop_at": 954, "target": 128747, "training/loss": 8.474327597022057, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.474327597022057, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.340679870401337, "training/generator_lr": 0.000499791775216379, "training/grad_norm": 0.20553956925868988, "training/positions": 587202560, "training/nonpadding_tokens": 587192883, "throughput/steps_per_second": 0.40957276074664617, "throughput/raw_tokens_per_second": 429457.96881093265, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 570, "stop_at": 954, "target": 128747, "training/loss": 8.484524634480476, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.484524634480476, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.26375842652592, "training/generator_lr": 0.0004997791602395484, "training/grad_norm": 0.22475376725196838, "training/positions": 597688320, "training/nonpadding_tokens": 597678457, "throughput/steps_per_second": 0.409565030912943, "throughput/raw_tokens_per_second": 429452.4439449952, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 580, "stop_at": 954, "target": 128747, "training/loss": 8.474682079255581, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.474682079255581, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.365311128239966, "training/generator_lr": 0.0004997661744602382, "training/grad_norm": 0.12845391035079956, "training/positions": 608174080, "training/nonpadding_tokens": 608164023, "throughput/steps_per_second": 0.40972923908611547, "throughput/raw_tokens_per_second": 429624.29785672436, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 590, "stop_at": 954, "target": 128747, "training/loss": 8.45921120867133, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.45921120867133, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.475081325773411, "training/generator_lr": 0.00049975281789812, "training/grad_norm": 0.22288911044597626, "training/positions": 618659840, "training/nonpadding_tokens": 618649723, "throughput/steps_per_second": 0.4097525531499001, "throughput/raw_tokens_per_second": 429654.23465639073, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 600, "stop_at": 954, "target": 128747, "training/loss": 8.452106051892041, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.452106051892041, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.514904825982441, "training/generator_lr": 0.0004997390905734268, "training/grad_norm": 0.1513429880142212, "training/positions": 629145600, "training/nonpadding_tokens": 629135483, "throughput/steps_per_second": 0.40972090053473176, "throughput/raw_tokens_per_second": 429623.5029991069, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 610, "stop_at": 954, "target": 128747, "training/loss": 8.443005571514368, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.443005571514368, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.594636744878763, "training/generator_lr": 0.0004997249925069534, "training/grad_norm": 0.19233877956867218, "training/positions": 639631360, "training/nonpadding_tokens": 639621155, "throughput/steps_per_second": 0.4096337113719802, "throughput/raw_tokens_per_second": 429528.4737589254, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 620, "stop_at": 954, "target": 128747, "training/loss": 8.435389909148217, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.435389909148217, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.628689054661371, "training/generator_lr": 0.0004997105237200563, "training/grad_norm": 0.2481939047574997, "training/positions": 650117120, "training/nonpadding_tokens": 650106618, "throughput/steps_per_second": 0.4097466443805617, "throughput/raw_tokens_per_second": 429638.3279026538, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 630, "stop_at": 954, "target": 128747, "training/loss": 8.432508453726768, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.432508453726768, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.666258949101171, "training/generator_lr": 0.0004996956842346532, "training/grad_norm": 0.21368791162967682, "training/positions": 660602880, "training/nonpadding_tokens": 660592301, "throughput/steps_per_second": 0.4096122489756663, "throughput/raw_tokens_per_second": 429506.41956759116, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 640, "stop_at": 954, "target": 128747, "training/loss": 8.418649353832006, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.418649353832006, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.74200133256689, "training/generator_lr": 0.0004996804740732238, "training/grad_norm": 0.24602854251861572, "training/positions": 671088640, "training/nonpadding_tokens": 671078026, "throughput/steps_per_second": 0.409638530688658, "throughput/raw_tokens_per_second": 429535.69822053285, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 650, "stop_at": 954, "target": 128747, "training/loss": 8.415336121618747, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.415336121618747, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.798717077759197, "training/generator_lr": 0.0004996648932588092, "training/grad_norm": 0.17990031838417053, "training/positions": 681574400, "training/nonpadding_tokens": 681563648, "throughput/steps_per_second": 0.4097179300578504, "throughput/raw_tokens_per_second": 429614.73412090575, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 660, "stop_at": 954, "target": 128747, "training/loss": 8.409692788124085, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.409692788124085, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.827396660744147, "training/generator_lr": 0.0004996489418150117, "training/grad_norm": 0.22107505798339844, "training/positions": 692060160, "training/nonpadding_tokens": 692049393, "throughput/steps_per_second": 0.4097667804936809, "throughput/raw_tokens_per_second": 429670.9969727712, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 670, "stop_at": 954, "target": 128747, "training/loss": 8.39473698735237, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.39473698735237, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.914405440008361, "training/generator_lr": 0.0004996326197659956, "training/grad_norm": 0.13812685012817383, "training/positions": 702545920, "training/nonpadding_tokens": 702535094, "throughput/steps_per_second": 0.40979924232373743, "throughput/raw_tokens_per_second": 429703.23250332556, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 680, "stop_at": 954, "target": 128747, "training/loss": 8.387701376527549, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.387701376527549, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.977392088210703, "training/generator_lr": 0.000499615927136486, "training/grad_norm": 0.25864216685295105, "training/positions": 713031680, "training/nonpadding_tokens": 713020784, "throughput/steps_per_second": 0.4098082724603821, "throughput/raw_tokens_per_second": 429712.25044551044, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 690, "stop_at": 954, "target": 128747, "training/loss": 8.38839731812477, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.38839731812477, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.984190465614549, "training/generator_lr": 0.0004995988639517697, "training/grad_norm": 0.16221806406974792, "training/positions": 723517440, "training/nonpadding_tokens": 723506441, "throughput/steps_per_second": 0.40975204620231753, "throughput/raw_tokens_per_second": 429651.9411525654, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 700, "stop_at": 954, "target": 128747, "training/loss": 8.376976038515568, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.376976038515568, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.07011163513796, "training/generator_lr": 0.0004995814302376948, "training/grad_norm": 0.21249906718730927, "training/positions": 734003200, "training/nonpadding_tokens": 733992123, "throughput/steps_per_second": 0.40979924914237803, "throughput/raw_tokens_per_second": 429702.46103457484, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 710, "stop_at": 954, "target": 128747, "training/loss": 8.375513978302479, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.375513978302479, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.093666714569398, "training/generator_lr": 0.0004995636260206707, "training/grad_norm": 0.26074185967445374, "training/positions": 744488960, "training/nonpadding_tokens": 744477778, "throughput/steps_per_second": 0.4097556596741445, "throughput/raw_tokens_per_second": 429655.64816404914, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 720, "stop_at": 954, "target": 128747, "training/loss": 8.362245783954858, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.362245783954858, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.153415029264214, "training/generator_lr": 0.000499545451327668, "training/grad_norm": 0.19179017841815948, "training/positions": 754974720, "training/nonpadding_tokens": 754963460, "throughput/steps_per_second": 0.40966374365972147, "throughput/raw_tokens_per_second": 429560.37429453555, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 730, "stop_at": 954, "target": 128747, "training/loss": 8.354028683900832, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.354028683900832, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.238908340301004, "training/generator_lr": 0.0004995269061862186, "training/grad_norm": 0.15193498134613037, "training/positions": 765460480, "training/nonpadding_tokens": 765449028, "throughput/steps_per_second": 0.4097851176955093, "throughput/raw_tokens_per_second": 429682.9716984266, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 740, "stop_at": 954, "target": 128747, "training/loss": 8.356391687691211, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.356391687691211, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.2392512803093645, "training/generator_lr": 0.0004995079906244155, "training/grad_norm": 0.27572599053382874, "training/positions": 775946240, "training/nonpadding_tokens": 775934485, "throughput/steps_per_second": 0.4097505663322511, "throughput/raw_tokens_per_second": 429642.19440024666, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 750, "stop_at": 954, "target": 128747, "training/loss": 8.343883656710386, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.343883656710386, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.299708337688127, "training/generator_lr": 0.0004994887046709126, "training/grad_norm": 0.184895321726799, "training/positions": 786432000, "training/nonpadding_tokens": 786420144, "throughput/steps_per_second": 0.4097135793488582, "throughput/raw_tokens_per_second": 429611.6880721569, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 760, "stop_at": 954, "target": 128747, "training/loss": 8.3386269249022, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.3386269249022, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.371502011914716, "training/generator_lr": 0.0004994690483549254, "training/grad_norm": 0.19373691082000732, "training/positions": 796917760, "training/nonpadding_tokens": 796905882, "throughput/steps_per_second": 0.4096756559120117, "throughput/raw_tokens_per_second": 429575.15928715054, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 770, "stop_at": 954, "target": 128747, "training/loss": 8.336367818713189, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.336367818713189, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.390949976484114, "training/generator_lr": 0.0004994490217062302, "training/grad_norm": 0.2017129808664322, "training/positions": 807403520, "training/nonpadding_tokens": 807391508, "throughput/steps_per_second": 0.40973699675985603, "throughput/raw_tokens_per_second": 429634.8906387062, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 780, "stop_at": 954, "target": 128747, "training/loss": 8.330749664455652, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.330749664455652, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.417807078281773, "training/generator_lr": 0.0004994286247551642, "training/grad_norm": 0.29035431146621704, "training/positions": 817889280, "training/nonpadding_tokens": 817877191, "throughput/steps_per_second": 0.4098039718594033, "throughput/raw_tokens_per_second": 429707.45410586236, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 790, "stop_at": 954, "target": 128747, "training/loss": 8.331660325080156, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.331660325080156, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.430668961642977, "training/generator_lr": 0.0004994078575326254, "training/grad_norm": 0.29904311895370483, "training/positions": 828375040, "training/nonpadding_tokens": 828362696, "throughput/steps_per_second": 0.409825764519385, "throughput/raw_tokens_per_second": 429723.01029968343, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 800, "stop_at": 954, "target": 128747, "training/loss": 8.322641830146313, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.322641830146313, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.496632655727424, "training/generator_lr": 0.0004993867200700733, "training/grad_norm": 0.14571811258792877, "training/positions": 838860800, "training/nonpadding_tokens": 838848410, "throughput/steps_per_second": 0.4098227516419064, "throughput/raw_tokens_per_second": 429728.41644100606, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 810, "stop_at": 954, "target": 128747, "training/loss": 8.310230723023414, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.310230723023414, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.584689851588629, "training/generator_lr": 0.0004993652123995276, "training/grad_norm": 0.20469273626804352, "training/positions": 849346560, "training/nonpadding_tokens": 849334036, "throughput/steps_per_second": 0.40963125023413727, "throughput/raw_tokens_per_second": 429524.0087867576, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 820, "stop_at": 954, "target": 128747, "training/loss": 8.312004368752241, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.312004368752241, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.595944816053512, "training/generator_lr": 0.0004993433345535693, "training/grad_norm": 0.33422917127609253, "training/positions": 859832320, "training/nonpadding_tokens": 859819502, "throughput/steps_per_second": 0.4096864064914724, "throughput/raw_tokens_per_second": 429575.2885928513, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 830, "stop_at": 954, "target": 128747, "training/loss": 8.309260212630033, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.309260212630033, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.614636679556856, "training/generator_lr": 0.0004993210865653397, "training/grad_norm": 0.17658238112926483, "training/positions": 870318080, "training/nonpadding_tokens": 870305154, "throughput/steps_per_second": 0.40970839244632357, "throughput/raw_tokens_per_second": 429605.96246715775, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 840, "stop_at": 954, "target": 128747, "training/loss": 8.297254644334316, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.297254644334316, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.708461799749164, "training/generator_lr": 0.0004992984684685413, "training/grad_norm": 0.2388930767774582, "training/positions": 880803840, "training/nonpadding_tokens": 880790680, "throughput/steps_per_second": 0.4096907636064629, "throughput/raw_tokens_per_second": 429582.31537554204, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 850, "stop_at": 954, "target": 128747, "training/loss": 8.291972026228905, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.291972026228905, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.723722630121237, "training/generator_lr": 0.0004992754802974368, "training/grad_norm": 0.26008257269859314, "training/positions": 891289600, "training/nonpadding_tokens": 891276364, "throughput/steps_per_second": 0.40977612147373454, "throughput/raw_tokens_per_second": 429678.29205191944, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 860, "stop_at": 954, "target": 128747, "training/loss": 8.29386702105403, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.29386702105403, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.738083651233278, "training/generator_lr": 0.0004992521220868498, "training/grad_norm": 0.2979568541049957, "training/positions": 901775360, "training/nonpadding_tokens": 901761946, "throughput/steps_per_second": 0.40968123226067327, "throughput/raw_tokens_per_second": 429574.6154730335, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 870, "stop_at": 954, "target": 128747, "training/loss": 8.29077036678791, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.29077036678791, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.77862242631689, "training/generator_lr": 0.0004992283938721643, "training/grad_norm": 0.1766432821750641, "training/positions": 912261120, "training/nonpadding_tokens": 912247625, "throughput/steps_per_second": 0.4096732490504494, "throughput/raw_tokens_per_second": 429570.2184430067, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 880, "stop_at": 954, "target": 128747, "training/loss": 8.2754708096385, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.2754708096385, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.878658026755853, "training/generator_lr": 0.0004992042956893251, "training/grad_norm": 0.2827114760875702, "training/positions": 922746880, "training/nonpadding_tokens": 922733261, "throughput/steps_per_second": 0.40967697499227573, "throughput/raw_tokens_per_second": 429572.3637350106, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 890, "stop_at": 954, "target": 128747, "training/loss": 8.275902789086103, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.275902789086103, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.892602620714883, "training/generator_lr": 0.0004991798275748369, "training/grad_norm": 0.26649755239486694, "training/positions": 933232640, "training/nonpadding_tokens": 933218837, "throughput/steps_per_second": 0.40974153952763087, "throughput/raw_tokens_per_second": 429637.6053073978, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 900, "stop_at": 954, "target": 128747, "training/loss": 8.274117908626796, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.274117908626796, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.905800911893813, "training/generator_lr": 0.0004991549895657653, "training/grad_norm": 0.23509182035923004, "training/positions": 943718400, "training/nonpadding_tokens": 943704386, "throughput/steps_per_second": 0.40984585625448794, "throughput/raw_tokens_per_second": 429745.88082033896, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 910, "stop_at": 954, "target": 128747, "training/loss": 8.263348757848144, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.263348757848144, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.9704255068979935, "training/generator_lr": 0.000499129781699736, "training/grad_norm": 0.2392078936100006, "training/positions": 954204160, "training/nonpadding_tokens": 954190088, "throughput/steps_per_second": 0.4099054671050202, "throughput/raw_tokens_per_second": 429814.6576234044, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 920, "stop_at": 954, "target": 128747, "training/loss": 8.263151784986258, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.263151784986258, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.006633439590301, "training/generator_lr": 0.000499104204014935, "training/grad_norm": 0.185787633061409, "training/positions": 964689920, "training/nonpadding_tokens": 964675810, "throughput/steps_per_second": 0.40982861796342074, "throughput/raw_tokens_per_second": 429734.8955608636, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 930, "stop_at": 954, "target": 128747, "training/loss": 8.256290387362242, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.256290387362242, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.0552639658235785, "training/generator_lr": 0.0004990782565501085, "training/grad_norm": 0.22627829015254974, "training/positions": 975175680, "training/nonpadding_tokens": 975161320, "throughput/steps_per_second": 0.40974839377523814, "throughput/raw_tokens_per_second": 429642.08804141975, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 940, "stop_at": 954, "target": 128747, "training/loss": 8.254573013260961, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.254573013260961, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.072237863189799, "training/generator_lr": 0.000499051939344563, "training/grad_norm": 0.27094876766204834, "training/positions": 985661440, "training/nonpadding_tokens": 985647032, "throughput/steps_per_second": 0.4097420771223371, "throughput/raw_tokens_per_second": 429643.7414986616, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 950, "stop_at": 954, "target": 128747, "training/loss": 8.251419907063246, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.251419907063246, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.108104488921405, "training/generator_lr": 0.0004990252524381648, "training/grad_norm": 0.22281892597675323, "training/positions": 996147200, "training/nonpadding_tokens": 996132624, "throughput/steps_per_second": 0.4096707425098597, "throughput/raw_tokens_per_second": 429564.0260295444, "throughput/peak_reserved_gib": 24.275390625}
+{"step": 954, "stop_at": 954, "target": 128747, "training/loss": 8.245156219229102, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.245156219229102, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.156394524717809, "training/generator_lr": 0.0004990144741684035, "training/grad_norm": 0.18212443590164185, "training/positions": 1000341504, "training/nonpadding_tokens": 1000326925, "throughput/steps_per_second": 0.4097792966142823, "throughput/raw_tokens_per_second": 429684.4283921452, "throughput/peak_reserved_gib": 24.275390625}
+{"validation_step": 954, "loss": 7.008299469947815, "reconstruction_loss": 0.0, "vq_loss": 0.0, "ncp_loss": 7.008299469947815, "mstok_loss": 0.0, "semantic_loss": 0.0, "ncp_accuracy": 17.890719063545152}
+wandb: - 0.022 MB of 0.022 MB uploaded
+wandb: 
+wandb: Run summary:
+wandb:     throughput/peak_reserved_gib 24.27539
+wandb: throughput/raw_tokens_per_second 429684.42839
+wandb:      throughput/steps_per_second 0.40978
+wandb:            training/generator_lr 0.0005
+wandb:               training/grad_norm 0.18212
+wandb:                    training/loss 8.24516
+wandb:              training/mstok_loss 0.0
+wandb:            training/ncp_accuracy 7.15639
+wandb:                training/ncp_loss 8.24516
+wandb:       training/nonpadding_tokens 1000326925
+wandb:               training/positions 1000341504
+wandb:     training/reconstruction_loss 0.0
+wandb:           training/semantic_loss 0.0
+wandb:                 training/vq_loss 0.0
+wandb:                  validation/loss 7.0083
+wandb:            validation/mstok_loss 0.0
+wandb:          validation/ncp_accuracy 17.89072
+wandb:              validation/ncp_loss 7.0083
+wandb:   validation/reconstruction_loss 0.0
+wandb:         validation/semantic_loss 0.0
+wandb:               validation/vq_loss 0.0
+wandb: 
+wandb:  View run iclr-debug-generator-decay27b-20260917-generator at: https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-generator-decay27b-20260917-generator
+wandb:  View project at: https://wandb.ai/mstok/iclr-debug
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20260917_222739-iclr-debug-generator-decay27b-20260917-generator/logs
+wandb: WARNING wandb version 0.30.0 is available!  To upgrade, please run:
+wandb: WARNING  $ pip install wandb --upgrade
+wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.

iclr-debug-generator-decay27b-20260917/generator/stopped-20260919/validation-curve.json

@@ -0,0 +1,432 @@
+[
+  {
+    "validation_step": 954,
+    "loss": 7.008299469947815,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 7.008299469947815,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 17.890719063545152
+  },
+  {
+    "validation_step": 1000,
+    "loss": 6.966006150245667,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.966006150245667,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 18.21814381270903
+  },
+  {
+    "validation_step": 2000,
+    "loss": 6.48116174697876,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.48116174697876,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 23.57458193979933
+  },
+  {
+    "validation_step": 3000,
+    "loss": 6.253293235301972,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.253293235301972,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 26.328177257525084
+  },
+  {
+    "validation_step": 4000,
+    "loss": 6.123198077678681,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.123198077678681,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 27.98829431438127
+  },
+  {
+    "validation_step": 4769,
+    "loss": 6.056744363307953,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.056744363307953,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 28.858612040133778
+  },
+  {
+    "validation_step": 5000,
+    "loss": 6.039546794891358,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 6.039546794891358,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 28.97015050167224
+  },
+  {
+    "validation_step": 6000,
+    "loss": 5.970354626178741,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.970354626178741,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 29.813127090301002
+  },
+  {
+    "validation_step": 7000,
+    "loss": 5.923566629886627,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.923566629886627,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 30.46923076923077
+  },
+  {
+    "validation_step": 8000,
+    "loss": 5.882993454933167,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.882993454933167,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 30.836371237458195
+  },
+  {
+    "validation_step": 9000,
+    "loss": 5.8473687791824345,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.8473687791824345,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 31.401003344481605
+  },
+  {
+    "validation_step": 10000,
+    "loss": 5.817564659118652,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.817564659118652,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 31.745484949832775
+  },
+  {
+    "validation_step": 11000,
+    "loss": 5.794896996021271,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.794896996021271,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.01914715719064
+  },
+  {
+    "validation_step": 12000,
+    "loss": 5.772765662670135,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.772765662670135,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.257023411371236
+  },
+  {
+    "validation_step": 12875,
+    "loss": 5.759038550853729,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.759038550853729,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.38352842809365
+  },
+  {
+    "validation_step": 13000,
+    "loss": 5.758613510131836,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.758613510131836,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.425501672240806
+  },
+  {
+    "validation_step": 14000,
+    "loss": 5.7420924544334415,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.7420924544334415,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.648494983277594
+  },
+  {
+    "validation_step": 15000,
+    "loss": 5.728990747928619,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.728990747928619,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 32.7757525083612
+  },
+  {
+    "validation_step": 16000,
+    "loss": 5.705576128959656,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.705576128959656,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.02207357859532
+  },
+  {
+    "validation_step": 17000,
+    "loss": 5.698980181217194,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.698980181217194,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.086622073578596
+  },
+  {
+    "validation_step": 18000,
+    "loss": 5.681112675666809,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.681112675666809,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.28051839464883
+  },
+  {
+    "validation_step": 19000,
+    "loss": 5.673746242523193,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.673746242523193,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.37299331103679
+  },
+  {
+    "validation_step": 20000,
+    "loss": 5.664321179389954,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.664321179389954,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.42775919732441
+  },
+  {
+    "validation_step": 21000,
+    "loss": 5.654858000278473,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.654858000278473,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.52792642140468
+  },
+  {
+    "validation_step": 22000,
+    "loss": 5.649980480670929,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.649980480670929,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.59556856187291
+  },
+  {
+    "validation_step": 23000,
+    "loss": 5.643369488716125,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.643369488716125,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.66371237458194
+  },
+  {
+    "validation_step": 24000,
+    "loss": 5.640048773288727,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.640048773288727,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.64088628762542
+  },
+  {
+    "validation_step": 25000,
+    "loss": 5.6390308260917665,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6390308260917665,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.712458193979934
+  },
+  {
+    "validation_step": 25750,
+    "loss": 5.63659159898758,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.63659159898758,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.715384615384615
+  },
+  {
+    "validation_step": 26000,
+    "loss": 5.636387121677399,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.636387121677399,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.697909698996654
+  },
+  {
+    "validation_step": 27000,
+    "loss": 5.636081528663635,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.636081528663635,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.701003344481606
+  },
+  {
+    "validation_step": 28000,
+    "loss": 5.634876639842987,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.634876639842987,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.72366220735786
+  },
+  {
+    "validation_step": 29000,
+    "loss": 5.633056950569153,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.633056950569153,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.727341137123744
+  },
+  {
+    "validation_step": 30000,
+    "loss": 5.631771521568298,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.631771521568298,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.72140468227425
+  },
+  {
+    "validation_step": 31000,
+    "loss": 5.6321802949905395,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6321802949905395,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.738127090301006
+  },
+  {
+    "validation_step": 32000,
+    "loss": 5.629039843082428,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.629039843082428,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.733946488294315
+  },
+  {
+    "validation_step": 33000,
+    "loss": 5.6280361485481265,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.6280361485481265,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.772157190635454
+  },
+  {
+    "validation_step": 34000,
+    "loss": 5.628514995574951,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.628514995574951,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.77851170568562
+  },
+  {
+    "validation_step": 35000,
+    "loss": 5.628999886512757,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.628999886512757,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.780685618729095
+  },
+  {
+    "validation_step": 36000,
+    "loss": 5.626763644218445,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.626763644218445,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.77943143812709
+  },
+  {
+    "validation_step": 37000,
+    "loss": 5.626788268089294,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.626788268089294,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.78929765886288
+  },
+  {
+    "validation_step": 38000,
+    "loss": 5.625574986934662,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.625574986934662,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.81563545150502
+  },
+  {
+    "validation_step": 39000,
+    "loss": 5.625601601600647,
+    "reconstruction_loss": 0.0,
+    "vq_loss": 0.0,
+    "ncp_loss": 5.625601601600647,
+    "mstok_loss": 0.0,
+    "semantic_loss": 0.0,
+    "ncp_accuracy": 33.80978260869565
+  }
+]