Back up decoder-aligned joint MsTok step 34333 with scheduled generation evaluation

iclr-debug-decoder-135b-20260916/decoder-mstok/checkpoint-34333-README.md

@@ -0,0 +1,15 @@
+# Decoder-aligned joint MsTok: step 34,333
+
+Full resumable checkpoint from `iclr-debug-decoder-135b-20260916`, trained jointly from scratch on full OWT.
+
+- 34,333 optimizer updates; 27,000,569,856 input positions including padding (27B milestone).
+- Context 256; 16 square scales; global batch 3,072; eight H100 workers.
+- Checkpoint includes online codec, generator, EMA teacher, semantic projector, optimizer, LR scheduler, eight per-rank RNG states, config, and non-padding token count.
+- Resolved config, run manifest, git provenance, and teacher provenance accompany this backup.
+- Generation evaluation: 640 samples across five seeds, supplied true level-zero code, temperature 1 with no top-k/top-p truncation. Reference model: GPT-2 Large in FP32 with TF32 enabled. These are the original scheduled evaluation results, not FP64 rescoring.
+- Generation PPL: 108.815 +/- 2.358 standard error; mean empirical token entropy: 4.372 nats.
+- 57.8% of passages contain Unicode replacement characters; adjacent repeated-word fraction: 3.56%. This intermediate research model still produces substantial generation defects.
+
+See `evaluation/step-34333/` for per-seed results and all saved samples. The full checkpoint is `milestone-iter-34333.pt`; component weights can be exported using `utils.iclr_training.export_training_checkpoint` in the matching source checkout. Saved absolute paths describe the original environment and are not portable defaults.
+
+The run-root `provenance/` directory contains the backup SHA-256 inventory. Training data and credentials are not included. Training continues independently of this snapshot.

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/EVAL_DONE.json

@@ -0,0 +1,5 @@
+{
+  "step": 34333,
+  "reference_dtype": "float32",
+  "sampling": "supplied-level0-untruncated"
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/backup-quality-summary.json

@@ -0,0 +1,10 @@
+{
+  "reference_dtype": "float32",
+  "tf32_allowed": true,
+  "scored_samples": 640,
+  "checkpoint_step": 34333,
+  "input_positions": 27000569856,
+  "samples_with_replacement_character": 370,
+  "fraction_samples_with_replacement_character": 0.578125,
+  "adjacent_repeated_word_fraction": 0.03556906253310112
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/summary.json

@@ -0,0 +1,35 @@
+{
+  "protocol": {
+    "sampling": "random",
+    "temperature": 1.0,
+    "top_k": 0,
+    "top_p": 1.0,
+    "n_provided_levels": 1,
+    "n_levels": 16,
+    "document_aware_validation_sampling": true
+  },
+  "reference_model": "gpt2-large",
+  "checkpoint": "/home/ubuntu/mstok-results/iclr-debug-decoder-135b-20260916/decoder-mstok/exports/step-34333/ncp.pt",
+  "checkpoint_step": 34333,
+  "seeds": [
+    0,
+    1,
+    2,
+    3,
+    4
+  ],
+  "samples_per_seed": 128,
+  "total_scored_samples": 640,
+  "total_scored_tokens": 160503,
+  "per_seed_mean_ppl": {
+    "0": 106.99428697044395,
+    "1": 104.0656312139604,
+    "2": 106.59998968944139,
+    "3": 117.78306621364587,
+    "4": 108.63302937787536
+  },
+  "mean_gen_ppl": 108.8152006930734,
+  "gen_ppl_se": 2.3583584883976796,
+  "mean_of_seed_median_ppl": 99.95965270996093,
+  "mean_entropy_nats": 4.37164044491404
+}

iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-34333.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d66c77a443ad8fbca0aaaa49b44de6f580ae4afbeda5c548635f22c6295e09
+size 4879455457

iclr-debug-decoder-135b-20260916/provenance/hf-backup-step-34333-manifest.json

@@ -0,0 +1,96 @@
+{
+  "repository": "iskhare/iclr-debug",
+  "checkpoint_step": 34333,
+  "files": [
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/milestone-iter-34333.pt",
+      "bytes": 4879455457,
+      "sha256": "f6d66c77a443ad8fbca0aaaa49b44de6f580ae4afbeda5c548635f22c6295e09"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/manifest.json",
+      "bytes": 28971,
+      "sha256": "53dc5b666685e03efc016e4311b89ab94bb6babe710405c28ae58356406582d0"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/git-provenance.json",
+      "bytes": 59,
+      "sha256": "96f601db3a051fd136086ca5f54c87ce2a72449e528d826a3b882541953e3fa2"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/config.yaml",
+      "bytes": 4827,
+      "sha256": "073bd5a69f1d0a39019efffd79f4b7bd645e0c1792edead7e31b524515ef134f"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/teacher-provenance.json",
+      "bytes": 270,
+      "sha256": "96e0063afe4211b1109509690a697fd8a7366941988c894f1027e1ead45938cf"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/summary.json",
+      "bytes": 863,
+      "sha256": "88c2c3e3e46a7dab61f4ffc02130ce28039ab501406e0f2a8383db3a1cc098ac"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/backup-quality-summary.json",
+      "bytes": 308,
+      "sha256": "182572770a6482956036192ef5f96a1de82f174e6e2ad5c585e0edab6527a81e"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/EVAL_DONE.json",
+      "bytes": 97,
+      "sha256": "434d7e1917506e6eed5b10f486506c86af623044870fa24b37fe21861c7e13db"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-0.json",
+      "bytes": 128859,
+      "sha256": "1d2e1212efc94dc3bc4240009f3aac1621ccfae37f2fb61522f1430dde7a6c52"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-0.samples.json",
+      "bytes": 129240,
+      "sha256": "6019c0d4abe0d80d536f2b31b6fd069b26356f2ebe249095a7794b72b50ecb85"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-1.json",
+      "bytes": 132204,
+      "sha256": "eedc81b617e82290cf09610beddbc671b2414a83f4d4bd5789c89641f0ff4b30"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-1.samples.json",
+      "bytes": 132586,
+      "sha256": "371350b51efcd33ff6f264b5b3f11c7b2ea9166926871050694599dddd13bc67"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-2.json",
+      "bytes": 132472,
+      "sha256": "6fd36561237c9029831b563aa35b3473281c4513baef5b2bef4ff8af54d64ac9"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-2.samples.json",
+      "bytes": 132853,
+      "sha256": "bf983134264fa81d9b13d539f4158bb50d2f2a7555be111a2d703be471c681b7"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-3.json",
+      "bytes": 130580,
+      "sha256": "677e40175f7df94d5ba18d7f24732a395fec8473c57f4dadd31ce13710dd583a"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-3.samples.json",
+      "bytes": 130961,
+      "sha256": "831d1faf708e672215788583247dda2cdedaee46bec3221fbcbaa25ff87b4911"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-4.json",
+      "bytes": 131641,
+      "sha256": "9e84e975a03b470f62c755f26cd9d53042d08ccae10e66737e1900ec971720be"
+    },
+    {
+      "path_in_repo": "iclr-debug-decoder-135b-20260916/decoder-mstok/evaluation/step-34333/random-seed-4.samples.json",
+      "bytes": 132021,
+      "sha256": "121b777e539813557f24ae1aae159741870b685158a51b7bcd26b9f6aa7a5749"
+    }
+  ]
+}