MsTok matched XSum joint-main-300k-20260922-v1-conditional: step 50002

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e3cdcf174cdc1fd991a854f0b1ca019a84d7d67dc46dc9eaea79b34a394f811
+size 2741342224

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/ema/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb7c3bbcd7bdc1da9a0cb1fc123ecee6eeb2ef456bc270c7619f8185c4c55134
+size 299639549

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/ema/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91bee359c7868f5e535798b48ce128a600fb06eeeced030bc1acea15a7eca2ac
+size 354177931

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/ema/metrics.json

@@ -0,0 +1,22 @@
+{
+  "step": 50002,
+  "variant": "ema",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.0361328125,
+  "released_inference_parity": null,
+  "self": {
+    "rouge1": 30.419240367687962,
+    "rouge2": 10.23110889369133,
+    "rougeL": 24.355588641077116,
+    "gen_ppl": 396.02394020492585
+  },
+  "oracle": {
+    "rouge1": 37.30270648532822,
+    "rouge2": 15.861188503038656,
+    "rougeL": 32.53096896323249,
+    "gen_ppl": 354.3321713544221
+  }
+}

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/manifest.json

@@ -0,0 +1,189 @@
+{
+  "config": {
+    "release_config": "config/final-release/xsum/conditional-gen.yaml",
+    "experiment": "mstok-matched-xsum",
+    "data_root": "/home/ishank/data/xsum-repro-20260921",
+    "seed": 42,
+    "codec_seed": 42,
+    "generator_seed": 55,
+    "global_batch": 512,
+    "total_updates": 50002,
+    "codec_lr": 0.0005,
+    "codec_min_lr": 5e-05,
+    "codec_warmup": 1000,
+    "codec_weight_decay": 0.01,
+    "generator_lr": 0.0003,
+    "generator_warmup": 200,
+    "generator_weight_decay": 0.1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "codebook_decay": 0.95,
+    "generator_ema_decay": 0.9999,
+    "checkpoint_interval": 2000,
+    "validation_interval": 1000,
+    "validation_batches": 8,
+    "neighbor_refresh_interval": 100,
+    "objectives": {
+      "codec_weight": 1.0,
+      "ncp_weight": 1.0,
+      "mstok_weight": 0.25,
+      "residual_weight": 1.0,
+      "reconstruction_weight": 1.0,
+      "soft_assignment_temperature": 1.0,
+      "prediction_temperature": 1.0,
+      "teacher_ema_decay": 0.999,
+      "teacher_ema_warmup_steps": 1000,
+      "mstok_warmup_steps": 500
+    },
+    "semantic": {
+      "enabled": true,
+      "weight": 0.5,
+      "warmup_steps": 500,
+      "teacher_dim": 768,
+      "projector_dim": 2048,
+      "projector_seed": 56,
+      "teacher_id": "FacebookAI/roberta-base",
+      "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+      "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e"
+    },
+    "resolved_codec": {
+      "n_layers": 6,
+      "embed_dim": 384,
+      "in_vocab_size": 50304,
+      "context_length": 48,
+      "compression_factor": 1,
+      "dropout": 0.1,
+      "pre_quant_groupnorm": 8,
+      "use_qk_norm": true,
+      "pre_quant_dropout": 0.25,
+      "vqvae_vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "vector_quantizer_config": {
+        "clss": "multiscale_residual_vector_quantizer",
+        "decay": 0.95,
+        "epsilon": 1e-05,
+        "commitment_cost": 0.25,
+        "learned_l1_sampling": true,
+        "learned_l1_cascade": true,
+        "learned_l1_cascade_base": 4,
+        "learned_all_sampling": false,
+        "quant_resi": {
+          "enabled": true,
+          "ratio": 0.5,
+          "share_mode": 0,
+          "learnable_ratio": false
+        },
+        "aux": {
+          "fine_drop": {
+            "prob": 0.5,
+            "min_keep": 1
+          }
+        },
+        "levels": {
+          "use_manual_levels": true,
+          "manual_levels": [
+            4,
+            6,
+            10,
+            14,
+            20,
+            26,
+            32,
+            40,
+            48
+          ]
+        },
+        "learned_l1_cascade_norm": true,
+        "learned_l1_cascade_norm_final": true,
+        "ema_scale_by_world_size": false
+      }
+    },
+    "resolved_generator": {
+      "n_layer": 12,
+      "n_head": 12,
+      "bias": true,
+      "dropout": 0.3,
+      "n_embd": 768,
+      "context_length": 560,
+      "prefix_len": 512,
+      "raw_token_prefix": true,
+      "prefix_pad_token_id": 50257,
+      "vocab_size": [
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096,
+        4096
+      ],
+      "predict_level0": false,
+      "attn_pattern": "block_diagonal",
+      "use_positional_encoding": true,
+      "use_level_encoding": true,
+      "use_rope": true,
+      "rope_base": 100000,
+      "use_qk_norm": true,
+      "ctx_encoder": {
+        "enabled": false,
+        "n_layer": 4,
+        "vocab_size": 50304
+      },
+      "use_flex_attention": false,
+      "query_level0": true,
+      "tie_heads_to_codebook": true,
+      "tie_heads_learn_scale": true
+    }
+  },
+  "runtime": {
+    "world": 8,
+    "microbatch": 32,
+    "accumulation": 2,
+    "deterministic": true,
+    "compile": true
+  },
+  "source_sha256": {
+    "scripts/train_matched_xsum.py": "f8b541d8e6776d8a6341d777c82e6d563f697130843ca04a2d5615f4031ca35c",
+    "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+    "trainer/mstok_trainer.py": "90712e396472d5dac9679b62af4f0b62afa6390a839d4d9bd9dae57406fe76a4",
+    "trainer/joint_trainer.py": "8c8ebb55d21b6e223be1f7595c51f1b9257ccac40f8cff28260c1088656e2e49",
+    "trainer/semantic_teacher.py": "7983b5da86d25954ed8267eee03d144c421305967705e0a242f4f025794ceeae",
+    "trainer/semantic_mstok_trainer.py": "ebc02e80d279a445c99d8f09505cdfaf7632f1a661d6880f09cc1bd0fa90ef30",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85"
+  },
+  "mode": "train",
+  "resumed_from": null,
+  "initialized_from_pretraining": {
+    "checkpoint": "/home/ishank/mstok/experiments/mstok-matched-xsum/joint-main-300k-20260922-v1/joint-main-300k-20260922-v1-pretrain/training/checkpoint-300000.pt",
+    "step": 300000,
+    "weights": "complete-pair deployment EMA",
+    "regenerated_buffers": [
+      "ncp.attn_mask_for_masking"
+    ],
+    "optimizer": "fresh",
+    "generator_ema_decay": 0.9999
+  },
+  "semantic_teacher": {
+    "teacher_id": "FacebookAI/roberta-base",
+    "teacher_revision": "e2da8e2f811d1448a5b465c236feacd80ffbac7b",
+    "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "token_map_sha256": "0d5aa4e0a98157722f03cf9a0c7a1f17178c370dd728896090077ea9efe0b593"
+  }
+}

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/raw/export/tokenizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8511c4c91d7a60479051051d607bb0693ea641c686c5ea40fee0073b7c4266a1
+size 299639549

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/raw/export/xsum.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba98f2f3bfd4fe1c6e7f7632bd60e404f71b8e57f00c1a3e1a377d2d829b1144
+size 354177931

mstok-matched-xsum/joint-main-300k-20260922-v1-conditional/step-50002/raw/metrics.json

@@ -0,0 +1,22 @@
+{
+  "step": 50002,
+  "variant": "raw",
+  "split": "validation",
+  "documents": 256,
+  "decoding": "greedy; predicted L0; 9 NFE",
+  "empty_predictions": 0,
+  "l0_code_accuracy": 0.009765625,
+  "released_inference_parity": null,
+  "self": {
+    "rouge1": 30.50013370220306,
+    "rouge2": 9.455983395905493,
+    "rougeL": 24.119169729307878,
+    "gen_ppl": 190.3869692959584
+  },
+  "oracle": {
+    "rouge1": 39.21345003893329,
+    "rouge2": 16.19865555394947,
+    "rougeL": 34.01795531605818,
+    "gen_ppl": 118.8333403796817
+  }
+}