MsTok matched XSum production-20260921-v1: step 2000

mstok-matched-xsum/production-20260921-v1/step-2000/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d30718a0f6ca7f78a9af0b1233228236a2eac6857811fc28331c16fda6324c03
+size 2741340713

mstok-matched-xsum/production-20260921-v1/step-2000/ema/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84008c9ffe9204088f5038aeffadf2205d0c005a8104169a8e67886622198449
+size 299639549

mstok-matched-xsum/production-20260921-v1/step-2000/ema/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af1d5207512b473e8fdc564729dd8a88b49d5243badbcaf2ec2b5b10e9e205ec
+size 354177867

mstok-matched-xsum/production-20260921-v1/step-2000/ema/metrics.json

@@ -0,0 +1,22 @@
+{
+  "step": 2000,
+  "variant": "ema",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.076171875,
+  "released_inference_parity": null,
+  "self": {
+    "rouge1": 0.4774449048092223,
+    "rouge2": 0.0,
+    "rougeL": 0.4774449048092223,
+    "gen_ppl": 11.684112973881453
+  },
+  "oracle": {
+    "rouge1": 0.18092491287562948,
+    "rouge2": 0.0,
+    "rougeL": 0.18092491287562948,
+    "gen_ppl": 225.07398386439345
+  }
+}

mstok-matched-xsum/production-20260921-v1/step-2000/manifest.json

@@ -0,0 +1,178 @@
+{
+  "config": {
+    "release_config": "config/final-release/xsum/conditional-gen.yaml",
+    "experiment": "mstok-matched-xsum",
+    "data_root": "/home/ishank/data/xsum-repro-20260921",
+    "seed": 42,
+    "codec_seed": 42,
+    "generator_seed": 55,
+    "global_batch": 512,
+    "total_updates": 50002,
+    "codec_lr": 0.0005,
+    "codec_min_lr": 5e-05,
+    "codec_warmup": 1000,
+    "codec_weight_decay": 0.01,
+    "generator_lr": 0.0003,
+    "generator_warmup": 200,
+    "generator_weight_decay": 0.1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "codebook_decay": 0.95,
+    "generator_ema_decay": 0.9999,
+    "checkpoint_interval": 2000,
+    "validation_interval": 1000,
+    "validation_batches": 8,
+    "neighbor_refresh_interval": 100,
+    "objectives": {
+      "codec_weight": 1.0,
+      "ncp_weight": 1.0,
+      "mstok_weight": 0.25,
+      "residual_weight": 1.0,
+      "reconstruction_weight": 1.0,
+      "soft_assignment_temperature": 1.0,
+      "prediction_temperature": 1.0,
+      "teacher_ema_decay": 0.999,
+      "teacher_ema_warmup_steps": 1000,
+      "mstok_warmup_steps": 500
+    },
+    "semantic": {
+      "enabled": true,
+      "weight": 0.5,
+      "warmup_steps": 500,
+      "teacher_dim": 768,
+      "projector_dim": 2048,
+      "projector_seed": 56,
+      "teacher_id": "FacebookAI/roberta-base",
+      "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    },
+    "resolved_codec": {
+      "n_layers": 6,
+      "embed_dim": 384,
+      "in_vocab_size": 50304,
+      "context_length": 48,
+      "compression_factor": 1,
+      "dropout": 0.1,
+      "pre_quant_groupnorm": 8,
+      "use_qk_norm": true,
+      "pre_quant_dropout": 0.25,
+      "vqvae_vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "vector_quantizer_config": {
+        "clss": "multiscale_residual_vector_quantizer",
+        "decay": 0.95,
+        "epsilon": 1e-05,
+        "commitment_cost": 0.25,
+        "learned_l1_sampling": true,
+        "learned_l1_cascade": true,
+        "learned_l1_cascade_base": 4,
+        "learned_all_sampling": false,
+        "quant_resi": {
+          "enabled": true,
+          "ratio": 0.5,
+          "share_mode": 0,
+          "learnable_ratio": false
+        },
+        "aux": {
+          "fine_drop": {
+            "prob": 0.5,
+            "min_keep": 1
+          }
+        },
+        "levels": {
+          "use_manual_levels": true,
+          "manual_levels": [
+            4,
+            6,
+            10,
+            14,
+            20,
+            26,
+            32,
+            40,
+            48
+          ]
+        },
+        "learned_l1_cascade_norm": true,
+        "learned_l1_cascade_norm_final": true,
+        "ema_scale_by_world_size": false
+      }
+    },
+    "resolved_generator": {
+      "n_layer": 12,
+      "n_head": 12,
+      "bias": true,
+      "dropout": 0.3,
+      "n_embd": 768,
+      "context_length": 560,
+      "prefix_len": 512,
+      "raw_token_prefix": true,
+      "prefix_pad_token_id": 50257,
+      "vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "predict_level0": false,
+      "attn_pattern": "block_diagonal",
+      "use_positional_encoding": true,
+      "use_level_encoding": true,
+      "use_rope": true,
+      "rope_base": 100000,
+      "use_qk_norm": true,
+      "ctx_encoder": {
+        "enabled": false,
+        "n_layer": 4,
+        "vocab_size": 50304
+      },
+      "use_flex_attention": false,
+      "query_level0": true,
+      "tie_heads_to_codebook": true,
+      "tie_heads_learn_scale": true
+    }
+  },
+  "runtime": {
+    "world": 8,
+    "microbatch": 32,
+    "accumulation": 2,
+    "deterministic": true,
+    "compile": true
+  },
+  "source_sha256": {
+    "scripts/train_matched_xsum.py": "09b0ba6d4d571d335685a2b877a4872ca1ac409f7213b821ce32e4deb114b02a",
+    "trainer/matched_xsum.py": "9410e5770ca947f58ea8db9c3ada0585ecf52d12d2366afdb192ddb866eb6cf1",
+    "trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
+    "trainer/joint_trainer.py": "8c8ebb55d21b6e223be1f7595c51f1b9257ccac40f8cff28260c1088656e2e49",
+    "trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
+    "trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/quant.py": "b7b73abd4e55e10454cb680dea82124c38c222d37236c4c76b5c7d21123ac1b1",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b"
+  },
+  "mode": "train",
+  "resumed_from": null,
+  "semantic_teacher": {
+    "teacher_id": "FacebookAI/roberta-base",
+    "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+    "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "token_map_sha256": "0d5aa4e0a98157722f03cf9a0c7a1f17178c370dd728896090077ea9efe0b593"
+  }
+}

mstok-matched-xsum/production-20260921-v1/step-2000/raw/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01b81439281acfe4ea0e3d9563e45b52df18917d7c6de16f5128a49a2a75a284
+size 299639549

mstok-matched-xsum/production-20260921-v1/step-2000/raw/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81e57083346d973b76ba80deef19f46f09af86615038adf0bca59729b5cb7c26
+size 354177867

mstok-matched-xsum/production-20260921-v1/step-2000/raw/metrics.json

@@ -0,0 +1,22 @@
+{
+  "step": 2000,
+  "variant": "raw",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.41015625,
+  "released_inference_parity": null,
+  "self": {
+    "rouge1": 17.37692310172764,
+    "rouge2": 1.7271381609086696,
+    "rougeL": 13.55548933101985,
+    "gen_ppl": 744.9358151059573
+  },
+  "oracle": {
+    "rouge1": 17.894921245433657,
+    "rouge2": 1.9490490069966324,
+    "rougeL": 14.138495997244599,
+    "gen_ppl": 761.4254895148869
+  }
+}