Save final step-128747 and best step-13000 tokenizer checkpoints

Full resumable states and standalone tokenizer exports, with resolved configurations, validation metrics, and SHA-256 manifest. Run iclr-debug-two-stage-135b-20260916.

iclr-debug-two-stage-135b-20260916/tokenizer/README.md

@@ -0,0 +1,14 @@
+# Tokenizer checkpoints: full OWT, context 256, 16 square scales
+
+Run: `iclr-debug-two-stage-135b-20260916`; source commit `1b5226d4bbdd34653cb5fdd58d436355fb76ac8c`.
+
+| Checkpoint | Input positions | Validation reconstruction loss | Full resumable state | Standalone tokenizer |
+|---|---:|---:|---|---|
+| Best validation, step 13,000 | 13,631,488,000 | 0.0005638275 | [checkpoint](checkpoint-iter-13000.pt) | [vqvae.pt](exports/step-13000/vqvae.pt) |
+| Final, step 128,747 | 135,001,014,272 | 0.0023283348 | [checkpoint](checkpoint-iter-128747.pt) | [vqvae.pt](exports/step-128747/vqvae.pt) |
+
+The codec uses GPT-2 input tokens, context 256, scale lengths 1² through 16², and 16,384 entries per codebook. Input-position budgets include padding. The manifest records actual non-padding token counts, validation metrics, file sizes and SHA-256 digests.
+
+The standalone files contain `model` (tokenizer state dict), `args` (codec configuration), `step`, and source provenance. Matching readable configuration YAMLs are included beside them. Full checkpoints also contain optimizer/scheduler state, per-rank RNG state, the resolved training configuration and token counters.
+
+The generator currently trains with the frozen final step-128747 tokenizer. Step 13000 is best by validation reconstruction loss; that selection does not measure generation quality. No generator weights or dataset files are included in this upload.

iclr-debug-two-stage-135b-20260916/tokenizer/backup-manifest.json

@@ -0,0 +1,121 @@
+{
+  "repo": "iskhare/iclr-debug",
+  "run": "iclr-debug-two-stage-135b-20260916",
+  "git_provenance": {
+    "commit": "1b5226d4bbdd34653cb5fdd58d436355fb76ac8c"
+  },
+  "checkpoints": [
+    {
+      "step": 13000,
+      "label": "best_validation_reconstruction",
+      "input_positions": 13631488000,
+      "nonpadding_tokens": 13631269884,
+      "validation": {
+        "validation_step": 13000,
+        "loss": 0.9200536286830903,
+        "reconstruction_loss": 0.0005638275156161399,
+        "vq_loss": 0.919489803314209,
+        "ncp_loss": 0.0,
+        "mstok_loss": 0.0,
+        "semantic_loss": 0.0,
+        "reconstruction_accuracy": 99.9931640625
+      },
+      "checkpoint": "iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-13000.pt",
+      "weights": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/vqvae.pt"
+    },
+    {
+      "step": 128747,
+      "label": "final",
+      "input_positions": 135001014272,
+      "nonpadding_tokens": 134998864457,
+      "validation": {
+        "validation_step": 128747,
+        "loss": 0.919041942358017,
+        "reconstruction_loss": 0.0023283348415861836,
+        "vq_loss": 0.9167136064171791,
+        "ncp_loss": 0.0,
+        "mstok_loss": 0.0,
+        "semantic_loss": 0.0,
+        "reconstruction_accuracy": 99.953125
+      },
+      "checkpoint": "iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-128747.pt",
+      "weights": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/vqvae.pt"
+    }
+  ],
+  "files": [
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/manual-checkpoints/checkpoint-iter-13000.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-13000.pt",
+      "size_bytes": 2285748353,
+      "sha256": "9dba57e01498f718f9b997ef90675044a4fdf35a36e3938b42d48aa142dd53cb"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/vqvae.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/vqvae.pt",
+      "size_bytes": 1300864085,
+      "sha256": "881095e365c818835e729265e0a6079920a9e6c1f5ae978a0767237dfdab09a2"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/hf-tokenizer-backup/vqvae-config-step-13000.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/config.yaml",
+      "size_bytes": 2204,
+      "sha256": "ed1886b9ebfc95c06cc51e9f32064569c79c114570b6fb1bfb0f7b1c4a5ed056"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/milestone-iter-128747.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-128747.pt",
+      "size_bytes": 2285748711,
+      "sha256": "0078f567fdf3b261aba666f8d591bedc32403b1c626eed9ef05c4f5c23a2cedd"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/vqvae.pt",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/vqvae.pt",
+      "size_bytes": 1300864085,
+      "sha256": "48f37a4e708a223adf92af3fe231a5886fe7bdbfd957d280e8dc10c66fa3dbc6"
+    },
+    {
+      "local_path": "/home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/hf-tokenizer-backup/vqvae-config-step-128747.yaml",
+      "repo_path": "iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/config.yaml",
+      "size_bytes": 2205,
+      "sha256": "ffd2387608c5b4880d8f5bafb5c4524e482953d9230c41eac8f3fe5a2b748eaf"
+    }
+  ],
+  "context_length": 256,
+  "scale_lengths": [
+    1,
+    4,
+    9,
+    16,
+    25,
+    36,
+    49,
+    64,
+    81,
+    100,
+    121,
+    144,
+    169,
+    196,
+    225,
+    256
+  ],
+  "codebook_sizes": [
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384,
+    16384
+  ],
+  "note": "Tokenizer encoder/quantizer/decoder only. The active generator uses final step 128747. Best checkpoint is selected by validation reconstruction loss, not generation quality."
+}

iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-128747.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0078f567fdf3b261aba666f8d591bedc32403b1c626eed9ef05c4f5c23a2cedd
+size 2285748711

iclr-debug-two-stage-135b-20260916/tokenizer/checkpoint-iter-13000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dba57e01498f718f9b997ef90675044a4fdf35a36e3938b42d48aa142dd53cb
+size 2285748353

iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-two-stage-135b-20260916-tokenizer
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-128747/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48f37a4e708a223adf92af3fe231a5886fe7bdbfd957d280e8dc10c66fa3dbc6
+size 1300864085

iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/config.yaml

@@ -0,0 +1,117 @@
+task: summarization
+experiment: iclr-debug-two-stage-135b-20260916-tokenizer
+experiment_dir: /home/ubuntu/mstok-results/iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000
+training:
+  seed: 42
+  log_interval: 10
+  eval_interval: 10000
+  checkpoint_interval: 10000
+  val_iters: 100
+  n_epochs: 10
+  total_iters: null
+  tqdm_interval: 10
+  batch_size: 256
+  eval_batch_size: 128
+  demo_batch_size: 4
+model:
+  n_layers: 6
+  context_length: 256
+  embed_dim: 384
+  in_vocab_size: 50304
+  vqvae_vocab_size:
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  - 16384
+  compression_factor: 1
+  dropout: 0.1
+  pre_quant_groupnorm: 4
+  pre_quant_dropout: 0.2
+  vector_quantizer:
+    clss: multiscale_residual_vector_quantizer
+    decay: 0.99
+    epsilon: 1.0e-05
+    commitment_cost: 0.25
+    learned_l1_sampling: true
+    learned_all_sampling: false
+    quant_resi:
+      enabled: true
+      ratio: 0.5
+      share_mode: 0
+      learnable_ratio: false
+    levels:
+      use_manual_levels: true
+      manual_levels:
+      - 1
+      - 4
+      - 9
+      - 16
+      - 25
+      - 36
+      - 49
+      - 64
+      - 81
+      - 100
+      - 121
+      - 144
+      - 169
+      - 196
+      - 225
+      - 256
+    aux:
+      fine_drop:
+        prob: 0.5
+        min_keep: 1
+wandb:
+  entity: mstok
+  project: owt-repro
+  enable: false
+  group: owtsmall-repro-ctx256-vqvae
+  id: owtsmall-repro-ctx256-vqvae-10ep
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.001
+  lr_decay_iters: null
+  min_lr: null
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 2
+torch_compile:
+  enable: false
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic
+dataset:
+  train_source: /home/ubuntu/data/full_owt/train_gpt2.bin
+  validate_source: /home/ubuntu/data/full_owt/valid_gpt2.bin
+  pad_token_id: 50257
+pre_tokenizer:
+  name: hf
+  tokenizer: hf
+  model_id: gpt2
+  special_tokens:
+    eos_token: <|endoftext|>
+    pad_token: <|pad|>

iclr-debug-two-stage-135b-20260916/tokenizer/exports/step-13000/vqvae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881095e365c818835e729265e0a6079920a9e6c1f5ae978a0767237dfdab09a2
+size 1300864085