xsum-guidance-adjacency5-1seed-20260923: full-test 1-seed guidance, adjacency=5.0

mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/REPORT.md

@@ -0,0 +1,9 @@
+# MsTok matched XSum: final raw checkpoint, full test
+
+11,233 human-reference test documents; 1 generation seed(s). Single-run scores; seed standard error is not estimated. Adjacency penalty=5.0, window=1, frequency penalty=0, applied to each candidate BEFORE MBR selection. Greedy seed variance is expected to be zero; it is not an estimate of training-seed or document uncertainty. No new De–En experiment was run.
+
+| Recipe | ROUGE-1 | ROUGE-2 | ROUGE-L | Gen PPL |
+|---|---:|---:|---:|---:|
+| guidance | 30.74 | 9.42 | 24.01 | 224.85 |
+
+MBR@16 uses temperature 0.5, top-p 0.9, no guidance, and the release fast unstemmed ROUGE-1 consensus utility. Guided greedy uses the release level-ramped w=0.5. Final scoring uses Porter-stemmed ROUGE against original references, not codec-reconstructed references. See results.json for per-seed scores, prediction parity, candidate diversity and precise values.

mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/guidance/seed-0/metrics.json

@@ -0,0 +1,9 @@
+{
+  "rouge1": 30.737785542001603,
+  "rouge2": 9.420765518446128,
+  "rougeL": 24.012409925991843,
+  "gen_ppl": 224.84736457112797,
+  "seed": 0,
+  "documents": 11233,
+  "empty_predictions": 1
+}

mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/manifest.json

@@ -0,0 +1,81 @@
+{
+  "step": 50002,
+  "weights": "raw",
+  "split": "test",
+  "documents": 11233,
+  "seeds": [
+    0
+  ],
+  "nshards": 8,
+  "sharding": "whole 32-document blocks, round-robin across GPUs",
+  "shard_rng": "generation_seed * 1000003 + shard_index",
+  "recipes": {
+    "greedy": {
+      "cfg": 0.0,
+      "temperature": 1.0,
+      "top_k": 1,
+      "top_p": 0.0,
+      "k": 1
+    },
+    "guidance": {
+      "cfg": 0.5,
+      "temperature": 1.0,
+      "top_k": 1,
+      "top_p": 0.0,
+      "k": 1
+    },
+    "mbr16": {
+      "cfg": 0.0,
+      "temperature": 0.5,
+      "top_k": 0,
+      "top_p": 0.9,
+      "k": 16
+    }
+  },
+  "frequency_penalty": 0.0,
+  "adjacency_penalty": 5.0,
+  "adjacency_window": 1,
+  "adjacency_decay": 1.0,
+  "penalty_applied": "every candidate before MBR selection",
+  "modes": [
+    "guidance"
+  ],
+  "paired_reference_root": null,
+  "mbr_utility": "unstemmed ROUGE-1 F consensus, excluding self; reference-free",
+  "final_rouge": "Porter-stemmed F, against original human summaries",
+  "standard_error": "not estimated: single generation seed",
+  "checkpoint": "/home/ishank/mstok/experiments/mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/checkpoint-50002.pt",
+  "qualification": {
+    "passed": true,
+    "checks": {
+      "documents": true,
+      "all_unpenalized_candidates_match": true,
+      "selection_uses_adjusted_candidates": true,
+      "penalty_changes_candidates": true,
+      "ppl_finite": true
+    },
+    "source_hashes": {
+      "scripts/evaluate_matched_xsum_recipes.py": "c3138e3f0bf5386e17247f71b156c0ea911b3bc30acacf8cbc661f4ba09eea10",
+      "scripts/launch_matched_xsum_recipes.py": "0e13cec2e6e3c11e8ae9c05c0b29ada421844eee47777082995b6e94cbbfce69",
+      "evaluation/xsum_mbr.py": "ef74cd2277fa6d851b068c30cef90d5f1a3f248846849a56388c261e5a7cbfa7",
+      "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+      "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+      "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+      "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+      "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+      "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+      "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+      "scripts/train_matched_xsum.py": "f8b541d8e6776d8a6341d777c82e6d563f697130843ca04a2d5615f4031ca35c",
+      "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85",
+      "evaluation/decoding_penalties.py": "70193076934ae8bf8cf6937f4d0d9acd46356bf20d770d3a18fdac46e652ddf0"
+    },
+    "unit_tests_passed": 15,
+    "documents": 32,
+    "seed": 0,
+    "adjacency_penalty": 5,
+    "frequency_penalty": 0,
+    "window": 1,
+    "changed_candidates": 347,
+    "candidates": 512
+  }
+}

mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/qualification.json

@@ -0,0 +1,33 @@
+{
+  "passed": true,
+  "checks": {
+    "documents": true,
+    "all_unpenalized_candidates_match": true,
+    "selection_uses_adjusted_candidates": true,
+    "penalty_changes_candidates": true,
+    "ppl_finite": true
+  },
+  "source_hashes": {
+    "scripts/evaluate_matched_xsum_recipes.py": "c3138e3f0bf5386e17247f71b156c0ea911b3bc30acacf8cbc661f4ba09eea10",
+    "scripts/launch_matched_xsum_recipes.py": "0e13cec2e6e3c11e8ae9c05c0b29ada421844eee47777082995b6e94cbbfce69",
+    "evaluation/xsum_mbr.py": "ef74cd2277fa6d851b068c30cef90d5f1a3f248846849a56388c261e5a7cbfa7",
+    "evaluation/gen_ppl.py": "e6853b14d27f93407505eb7607a1bd9fed49a7add1511ceca2b4a3076d533859",
+    "models/next_concept.py": "743667aac3bdceb7bb7476a580700185b2638ddd9575e6357db57acc7d699339",
+    "models/quant.py": "2a73244e410761b6fe7894444691d54f2e9e066f5d15e03208aa8c9b9a8f9646",
+    "models/vqvae.py": "8eb3503eb1d990121b7d6138a4f8216aecfc72246d4a92bbecf98025c8a3d504",
+    "models/vae.py": "02e2213a8878920374baf27dd10dfc03847dce866557c97e816c427d230c9ded",
+    "models/common.py": "99aa0da356963a282936413d738ee1da72a3e3f277fecc28eddb553051582a4b",
+    "trainer/matched_xsum.py": "b657e3d21cc863a4395b3495ecdafd0349405b64d2159bfaa91efed8f2e285d4",
+    "scripts/train_matched_xsum.py": "f8b541d8e6776d8a6341d777c82e6d563f697130843ca04a2d5615f4031ca35c",
+    "data/matched_xsum_pretrain.py": "45380fd6d38b82b4ef3d11578a013356fa7f1e817ff8e90ec5d42884b0469d85",
+    "evaluation/decoding_penalties.py": "70193076934ae8bf8cf6937f4d0d9acd46356bf20d770d3a18fdac46e652ddf0"
+  },
+  "unit_tests_passed": 15,
+  "documents": 32,
+  "seed": 0,
+  "adjacency_penalty": 5,
+  "frequency_penalty": 0,
+  "window": 1,
+  "changed_candidates": 347,
+  "candidates": 512
+}

mstok-matched-xsum/xsum-guidance-adjacency5-1seed-20260923/results.json

@@ -0,0 +1,36 @@
+{
+  "guidance": {
+    "per_seed": [
+      {
+        "rouge1": 30.737785542001603,
+        "rouge2": 9.420765518446128,
+        "rougeL": 24.012409925991843,
+        "gen_ppl": 224.84736457112797,
+        "seed": 0,
+        "documents": 11233,
+        "empty_predictions": 1
+      }
+    ],
+    "summary": {
+      "rouge1": {
+        "mean": 30.737785542001603,
+        "standard_error": null
+      },
+      "rouge2": {
+        "mean": 9.420765518446128,
+        "standard_error": null
+      },
+      "rougeL": {
+        "mean": 24.012409925991843,
+        "standard_error": null
+      },
+      "gen_ppl": {
+        "mean": 224.84736457112797,
+        "standard_error": null
+      }
+    },
+    "differing_predictions_vs_seed0": [
+      0
+    ]
+  }
+}