MsTok matched XSum gentle-codec-20260923-v1-conditional: step 18000

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5841880f46985216c0964f93c7cafe7c718603060528fa524705193c3517908
+size 2741342544

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/ema/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9cd670b67a8dd395404edb8ce3b3f66e18ed6979498386491b209046caa309
+size 299639549

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/ema/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac12d3fb837734f779091362a3e4cdc5b2a2b45cb3c6edd128482a45dbe82ee
+size 354177931

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/ema/metrics.json

@@ -0,0 +1,28 @@
+{
+  "step": 18000,
+  "variant": "ema",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.001953125,
+  "released_inference_parity": null,
+  "token_decoding_penalties": {
+    "frequency": 0.0,
+    "adjacency": 0.0,
+    "window": 1,
+    "decay": 1.0
+  },
+  "self": {
+    "rouge1": 22.09098242622395,
+    "rouge2": 5.687127286596842,
+    "rougeL": 17.517762212946046,
+    "gen_ppl": 269.5198885759897
+  },
+  "oracle": {
+    "rouge1": 34.3593632826079,
+    "rouge2": 14.209334365515764,
+    "rougeL": 30.36523920383688,
+    "gen_ppl": 186.40590260250394
+  }
+}

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/manifest.json

@@ -0,0 +1,195 @@
+{
+  "config": {
+    "release_config": "config/final-release/xsum/conditional-gen.yaml",
+    "experiment": "mstok-matched-xsum-gentle-codec",
+    "data_root": "/home/ishank/data/xsum-repro-20260921",
+    "seed": 42,
+    "codec_seed": 42,
+    "generator_seed": 55,
+    "global_batch": 512,
+    "total_updates": 50002,
+    "codec_lr": 5e-05,
+    "codec_min_lr": 5e-06,
+    "codec_warmup": 1000,
+    "codec_weight_decay": 0.01,
+    "projector_lr": 0.0005,
+    "projector_min_lr": 5e-05,
+    "projector_warmup": 1000,
+    "projector_weight_decay": 0.01,
+    "generator_lr": 0.0003,
+    "generator_warmup": 200,
+    "generator_weight_decay": 0.1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "codebook_decay": 0.95,
+    "generator_ema_decay": 0.9999,
+    "checkpoint_interval": 2000,
+    "validation_interval": 1000,
+    "validation_batches": 8,
+    "neighbor_refresh_interval": 100,
+    "ddp_fixed_buckets": true,
+    "objectives": {
+      "codec_weight": 1.0,
+      "ncp_weight": 1.0,
+      "mstok_weight": 0.25,
+      "residual_weight": 1.0,
+      "reconstruction_weight": 1.0,
+      "soft_assignment_temperature": 1.0,
+      "prediction_temperature": 1.0,
+      "teacher_ema_decay": 0.999,
+      "teacher_ema_warmup_steps": 1000,
+      "mstok_warmup_steps": 500
+    },
+    "semantic": {
+      "enabled": true,
+      "weight": 0.5,
+      "warmup_steps": 500,
+      "teacher_dim": 768,
+      "projector_dim": 2048,
+      "projector_seed": 56,
+      "teacher_id": "FacebookAI/roberta-base",
+      "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    },
+    "resolved_codec": {
+      "n_layers": 6,
+      "embed_dim": 384,
+      "in_vocab_size": 50304,
+      "context_length": 48,
+      "compression_factor": 1,
+      "dropout": 0.1,
+      "pre_quant_groupnorm": 8,
+      "use_qk_norm": true,
+      "pre_quant_dropout": 0.25,
+      "vqvae_vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "vector_quantizer_config": {
+        "clss": "multiscale_residual_vector_quantizer",
+        "decay": 0.95,
+        "epsilon": 1e-05,
+        "commitment_cost": 0.25,
+        "learned_l1_sampling": true,
+        "learned_l1_cascade": true,
+        "learned_l1_cascade_base": 4,
+        "learned_all_sampling": false,
+        "quant_resi": {
+          "enabled": true,
+          "ratio": 0.5,
+          "share_mode": 0,
+          "learnable_ratio": false
+        },
+        "aux": {
+          "fine_drop": {
+            "prob": 0.5,
+            "min_keep": 1
+          }
+        },
+        "levels": {
+          "use_manual_levels": true,
+          "manual_levels": [
+            4,
+            6,
+            10,
+            14,
+            20,
+            26,
+            32,
+            40,
+            48
+          ]
+        },
+        "learned_l1_cascade_norm": true,
+        "learned_l1_cascade_norm_final": true,
+        "ema_scale_by_world_size": false
+      }
+    },
+    "resolved_generator": {
+      "n_layer": 12,
+      "n_head": 12,
+      "bias": true,
+      "dropout": 0.3,
+      "n_embd": 768,
+      "context_length": 560,
+      "prefix_len": 512,
+      "raw_token_prefix": true,
+      "prefix_pad_token_id": 50257,
+      "vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "predict_level0": false,
+      "attn_pattern": "block_diagonal",
+      "use_positional_encoding": true,
+      "use_level_encoding": true,
+      "use_rope": true,
+      "rope_base": 100000,
+      "use_qk_norm": true,
+      "ctx_encoder": {
+        "enabled": false,
+        "n_layer": 4,
+        "vocab_size": 50304
+      },
+      "use_flex_attention": false,
+      "query_level0": true,
+      "tie_heads_to_codebook": true,
+      "tie_heads_learn_scale": true
+    }
+  },
+  "runtime": {
+    "world": 8,
+    "microbatch": 32,
+    "accumulation": 2,
+    "deterministic": true,
+    "ddp_fixed_buckets": true,
+    "compile": true
+  },
+  "source_sha256": {
+    "scripts/train_matched_xsum.py": "60e18d0160e8378aaf44daf51c80167968e469130e4bdbdfeda78c7826f41561",
+    "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+    "trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
+    "trainer/joint_trainer.py": "8c8ebb55d21b6e223be1f7595c51f1b9257ccac40f8cff28260c1088656e2e49",
+    "trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
+    "trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85"
+  },
+  "mode": "train",
+  "resumed_from": null,
+  "initialized_from_pretraining": {
+    "checkpoint": "/home/ishank/mstok/experiments/mstok-matched-xsum/joint-main-300k-20260922-v1/joint-main-300k-20260922-v1-pretrain/training/checkpoint-300000.pt",
+    "step": 300000,
+    "weights": "complete-pair deployment EMA",
+    "regenerated_buffers": [
+      "ncp.attn_mask_for_masking"
+    ],
+    "optimizer": "fresh",
+    "generator_ema_decay": 0.9999
+  },
+  "semantic_teacher": {
+    "teacher_id": "FacebookAI/roberta-base",
+    "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+    "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "token_map_sha256": "0d5aa4e0a98157722f03cf9a0c7a1f17178c370dd728896090077ea9efe0b593"
+  }
+}

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/raw/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ac67299b8e4618e2cab76c0a628131a84de8ad5dc3f96a4258a09848235cef6
+size 299639549

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/raw/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e593d8a1dc747a6dbf7300df8c209c14f797ce1dd9bcfb4f253cfe50eea482b
+size 354177931

mstok-matched-xsum/gentle-codec-20260923-v1-conditional/step-18000/raw/metrics.json

@@ -0,0 +1,28 @@
+{
+  "step": 18000,
+  "variant": "raw",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.0029296875,
+  "released_inference_parity": null,
+  "token_decoding_penalties": {
+    "frequency": 0.0,
+    "adjacency": 0.0,
+    "window": 1,
+    "decay": 1.0
+  },
+  "self": {
+    "rouge1": 29.698298575540242,
+    "rouge2": 8.581683651563305,
+    "rougeL": 23.58177844948194,
+    "gen_ppl": 181.68185006559435
+  },
+  "oracle": {
+    "rouge1": 38.56363998130119,
+    "rouge2": 16.06303714051799,
+    "rougeL": 33.713701709276535,
+    "gen_ppl": 114.68465784047598
+  }
+}