gentle-codec-20260923-v1-final-recipes-adjacency5: full-test 1-seed greedy,guidance,mbr16, adjacency=5.0

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 mstok-matched-xsum/xsum-table-1seed-20260923/mbr16/seed-0/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
 mstok-matched-xsum/xsum-mbr16-adjacency5-1seed-20260923/mbr16/seed-0/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
+mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/mbr16/seed-0/predictions.jsonl filter=lfs diff=lfs merge=lfs -text

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/REPORT.md

@@ -0,0 +1,11 @@
+# MsTok matched XSum: final raw checkpoint, full test
+
+11,233 human-reference test documents; 1 generation seed(s). Single-run scores; seed standard error is not estimated. Adjacency penalty=5.0, window=1, frequency penalty=0, applied to each candidate BEFORE MBR selection. Greedy seed variance is expected to be zero; it is not an estimate of training-seed or document uncertainty. No new De–En experiment was run.
+
+| Recipe | ROUGE-1 | ROUGE-2 | ROUGE-L | Gen PPL |
+|---|---:|---:|---:|---:|
+| greedy | 29.85 | 9.04 | 23.39 | 207.48 |
+| guidance | 30.43 | 9.15 | 23.65 | 247.10 |
+| mbr16 | 29.84 | 7.64 | 22.09 | 391.77 |
+
+MBR@16 uses temperature 0.5, top-p 0.9, no guidance, and the release fast unstemmed ROUGE-1 consensus utility. Guided greedy uses the release level-ramped w=0.5. Final scoring uses Porter-stemmed ROUGE against original references, not codec-reconstructed references. See results.json for per-seed scores, prediction parity, candidate diversity and precise values.

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/greedy/seed-0/metrics.json

@@ -0,0 +1,10 @@
+{
+  "rouge1": 29.85257014508451,
+  "rouge2": 9.04211817927582,
+  "rougeL": 23.38604218759476,
+  "gen_ppl": 207.47876685542496,
+  "seed": 0,
+  "documents": 11233,
+  "empty_predictions": 1,
+  "differences_from_existing_baseline": 6176
+}

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/guidance/seed-0/metrics.json

@@ -0,0 +1,9 @@
+{
+  "rouge1": 30.42566100620435,
+  "rouge2": 9.153017688411932,
+  "rougeL": 23.65170563464576,
+  "gen_ppl": 247.09878065399937,
+  "seed": 0,
+  "documents": 11233,
+  "empty_predictions": 0
+}

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/manifest.json

@@ -0,0 +1,76 @@
+{
+  "step": 50002,
+  "weights": "raw",
+  "split": "test",
+  "documents": 11233,
+  "seeds": [
+    0
+  ],
+  "nshards": 8,
+  "sharding": "whole 32-document blocks, round-robin across GPUs",
+  "shard_rng": "generation_seed * 1000003 + shard_index",
+  "recipes": {
+    "greedy": {
+      "cfg": 0.0,
+      "temperature": 1.0,
+      "top_k": 1,
+      "top_p": 0.0,
+      "k": 1
+    },
+    "guidance": {
+      "cfg": 0.5,
+      "temperature": 1.0,
+      "top_k": 1,
+      "top_p": 0.0,
+      "k": 1
+    },
+    "mbr16": {
+      "cfg": 0.0,
+      "temperature": 0.5,
+      "top_k": 0,
+      "top_p": 0.9,
+      "k": 16
+    }
+  },
+  "frequency_penalty": 0.0,
+  "adjacency_penalty": 5.0,
+  "adjacency_window": 1,
+  "adjacency_decay": 1.0,
+  "penalty_applied": "every candidate before MBR selection",
+  "modes": [
+    "greedy",
+    "guidance",
+    "mbr16"
+  ],
+  "paired_reference_root": null,
+  "mbr_utility": "unstemmed ROUGE-1 F consensus, excluding self; reference-free",
+  "final_rouge": "Porter-stemmed F, against original human summaries",
+  "standard_error": "not estimated: single generation seed",
+  "checkpoint": "/home/ishank/mstok/experiments/mstok-matched-xsum/gentle-codec-20260923-v1/gentle-codec-20260923-v1-final-recipes-adjacency5/checkpoint-50002.pt",
+  "qualification": {
+    "passed": true,
+    "source_hashes": {
+      "scripts/evaluate_matched_xsum_recipes.py": "c3138e3f0bf5386e17247f71b156c0ea911b3bc30acacf8cbc661f4ba09eea10",
+      "scripts/launch_matched_xsum_recipes.py": "0e13cec2e6e3c11e8ae9c05c0b29ada421844eee47777082995b6e94cbbfce69",
+      "evaluation/xsum_mbr.py": "ef74cd2277fa6d851b068c30cef90d5f1a3f248846849a56388c261e5a7cbfa7",
+      "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+      "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+      "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+      "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+      "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+      "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+      "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+      "scripts/train_matched_xsum.py": "60e18d0160e8378aaf44daf51c80167968e469130e4bdbdfeda78c7826f41561",
+      "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85",
+      "evaluation/decoding_penalties.py": "70193076934ae8bf8cf6937f4d0d9acd46356bf20d770d3a18fdac46e652ddf0"
+    },
+    "checks": {
+      "previous_inference_qualification_passed": true,
+      "only_training_script_changed": true,
+      "evaluation_imports_unchanged": true,
+      "generator_ema_unchanged": true
+    },
+    "evidence": "Reuses completed full-test inference verification. Training optimizer/LR grouping, DDP setup, and logs changed; evaluation imports dataset/json_save remain AST-identical. All model/evaluator/penalty/MBR source hashes unchanged.",
+    "previous_evidence": "experiments/mstok-matched-xsum/mbr-adjacency5-qualification-20260923/qualification.json"
+  }
+}

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/mbr16/seed-0/metrics.json

@@ -0,0 +1,10 @@
+{
+  "rouge1": 29.838501266110686,
+  "rouge2": 7.640362235391838,
+  "rougeL": 22.089392631123275,
+  "gen_ppl": 391.76932941394034,
+  "seed": 0,
+  "documents": 11233,
+  "empty_predictions": 0,
+  "mean_distinct_candidates": 15.984509926110567
+}

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/mbr16/seed-0/predictions.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a359217b7a741c090d72cf25bfd3da6ec4ec7e85ce946c71e46dfaa014412a1d
+size 33040644

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/qualification.json

@@ -0,0 +1,26 @@
+{
+  "passed": true,
+  "source_hashes": {
+    "scripts/evaluate_matched_xsum_recipes.py": "c3138e3f0bf5386e17247f71b156c0ea911b3bc30acacf8cbc661f4ba09eea10",
+    "scripts/launch_matched_xsum_recipes.py": "0e13cec2e6e3c11e8ae9c05c0b29ada421844eee47777082995b6e94cbbfce69",
+    "evaluation/xsum_mbr.py": "ef74cd2277fa6d851b068c30cef90d5f1a3f248846849a56388c261e5a7cbfa7",
+    "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+    "scripts/train_matched_xsum.py": "60e18d0160e8378aaf44daf51c80167968e469130e4bdbdfeda78c7826f41561",
+    "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85",
+    "evaluation/decoding_penalties.py": "70193076934ae8bf8cf6937f4d0d9acd46356bf20d770d3a18fdac46e652ddf0"
+  },
+  "checks": {
+    "previous_inference_qualification_passed": true,
+    "only_training_script_changed": true,
+    "evaluation_imports_unchanged": true,
+    "generator_ema_unchanged": true
+  },
+  "evidence": "Reuses completed full-test inference verification. Training optimizer/LR grouping, DDP setup, and logs changed; evaluation imports dataset/json_save remain AST-identical. All model/evaluator/penalty/MBR source hashes unchanged.",
+  "previous_evidence": "experiments/mstok-matched-xsum/mbr-adjacency5-qualification-20260923/qualification.json"
+}

mstok-matched-xsum/gentle-codec-20260923-v1-final-recipes-adjacency5/results.json

@@ -0,0 +1,106 @@
+{
+  "greedy": {
+    "per_seed": [
+      {
+        "rouge1": 29.85257014508451,
+        "rouge2": 9.04211817927582,
+        "rougeL": 23.38604218759476,
+        "gen_ppl": 207.47876685542496,
+        "seed": 0,
+        "documents": 11233,
+        "empty_predictions": 1,
+        "differences_from_existing_baseline": 6176
+      }
+    ],
+    "summary": {
+      "rouge1": {
+        "mean": 29.85257014508451,
+        "standard_error": null
+      },
+      "rouge2": {
+        "mean": 9.04211817927582,
+        "standard_error": null
+      },
+      "rougeL": {
+        "mean": 23.38604218759476,
+        "standard_error": null
+      },
+      "gen_ppl": {
+        "mean": 207.47876685542496,
+        "standard_error": null
+      }
+    },
+    "differing_predictions_vs_seed0": [
+      0
+    ]
+  },
+  "guidance": {
+    "per_seed": [
+      {
+        "rouge1": 30.42566100620435,
+        "rouge2": 9.153017688411932,
+        "rougeL": 23.65170563464576,
+        "gen_ppl": 247.09878065399937,
+        "seed": 0,
+        "documents": 11233,
+        "empty_predictions": 0
+      }
+    ],
+    "summary": {
+      "rouge1": {
+        "mean": 30.42566100620435,
+        "standard_error": null
+      },
+      "rouge2": {
+        "mean": 9.153017688411932,
+        "standard_error": null
+      },
+      "rougeL": {
+        "mean": 23.65170563464576,
+        "standard_error": null
+      },
+      "gen_ppl": {
+        "mean": 247.09878065399937,
+        "standard_error": null
+      }
+    },
+    "differing_predictions_vs_seed0": [
+      0
+    ]
+  },
+  "mbr16": {
+    "per_seed": [
+      {
+        "rouge1": 29.838501266110686,
+        "rouge2": 7.640362235391838,
+        "rougeL": 22.089392631123275,
+        "gen_ppl": 391.76932941394034,
+        "seed": 0,
+        "documents": 11233,
+        "empty_predictions": 0,
+        "mean_distinct_candidates": 15.984509926110567
+      }
+    ],
+    "summary": {
+      "rouge1": {
+        "mean": 29.838501266110686,
+        "standard_error": null
+      },
+      "rouge2": {
+        "mean": 7.640362235391838,
+        "standard_error": null
+      },
+      "rougeL": {
+        "mean": 22.089392631123275,
+        "standard_error": null
+      },
+      "gen_ppl": {
+        "mean": 391.76932941394034,
+        "standard_error": null
+      }
+    },
+    "differing_predictions_vs_seed0": [
+      0
+    ]
+  }
+}