Reorganize Apertus tokenizer extension release

Add curated tokenizer/checkpoint/dataset/results layout, manifests, code links, and provenance pointers. No legacy paths deleted.

.gitattributes

@@ -39,3 +39,12 @@ subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_ex
 subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_per_task_van_td.png filter=lfs diff=lfs merge=lfs -text
 subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_van_td.png filter=lfs diff=lfs merge=lfs -text
 subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_van_td_gap.png filter=lfs diff=lfs merge=lfs -text
+results/plots/global_mmlu_full_el_subcategories_van_td.png filter=lfs diff=lfs merge=lfs -text
+results/plots/include_base_44_greek_subjects_van_td.png filter=lfs diff=lfs merge=lfs -text
+results/plots/intrinsic_trajectories.png filter=lfs diff=lfs merge=lfs -text
+results/plots/intrinsic_van_td.png filter=lfs diff=lfs merge=lfs -text
+results/plots/trajectories.png filter=lfs diff=lfs merge=lfs -text
+results/plots/trajectories_per_task.png filter=lfs diff=lfs merge=lfs -text
+results/plots/trajectories_per_task_van_td.png filter=lfs diff=lfs merge=lfs -text
+results/plots/trajectories_van_td.png filter=lfs diff=lfs merge=lfs -text
+results/plots/trajectories_van_td_gap.png filter=lfs diff=lfs merge=lfs -text

ARTIFACT_GRAPH.md

@@ -0,0 +1,51 @@
+# Artifact Graph
+
+```text
+swiss-ai/Apertus-8B-2509
+  |
+  | continuous BPE on GlossAPI + HPLT 50/50
+  v
+C3 full candidate
+  base 131072 + 25600 = 156672
+  |
+  | cutoff sweep, curation, backfill, alignment
+  v
+tokenizer/modern-greek-17408
+  base 131072 + 17408 = 148480
+  |
+  | optional polytonic stack
+  v
+tokenizer/polytonic-plus-5120
+  base 131072 + modern 17408 + polytonic 5120 = 153600
+
+fffoivos/glossapi-greek-nanochat-pretraining-dataset
+  + nanochat internal dedup metadata
+  + Apertus overlap drop overlay
+  |
+  v
+selected_after_apertus_and_internal_dedup.parquet
+  |
+  | 70/24/4/2 Greek/replay/code/math mix
+  v
+bulk_mix.nfc.jsonl
+  |
+  | Megatron indexed dataset preprocessing
+  v
+bulk_mix_base_nfc_megatron/bulk_mix_text_document.{bin,idx}
+
+modern tokenizer + CPT mix
+  |
+  | Vanilla, ReTok, Centroid, Token Distillation layer 11
+  v
+2B bakeoff and 3.5B continuation
+  |
+  v
+checkpoints/td-layer11-cpt-3p5b-iter834
+  |
+  v
+results/
+```
+
+The large checkpoint and dataset payloads are represented by hydration pointers
+unless explicitly uploaded as model weights.
+

MANIFEST.json

@@ -0,0 +1,150 @@
+{
+  "release_name": "apertus-tokenizer-extension",
+  "generated_date": "2026-05-25",
+  "source_repo": "https://github.com/fffoivos/glossapi-tokenizer-extension",
+  "source_subproject": "subprojects/03_apertus_extension_and_embedding_adaptation",
+  "source_commit_for_code_refs": "7c67cf11eb3d31f0ddd006f7d709709037db93a1",
+  "artifacts": [
+    {
+      "artifact_id": "tokenizer.modern-greek-17408",
+      "type": "tokenizer",
+      "status": "payload-present",
+      "public_path": "tokenizer/modern-greek-17408",
+      "base_model": "swiss-ai/Apertus-8B-2509",
+      "vocab_size": 148480,
+      "base_vocab_size": 131072,
+      "added_modern_greek_tokens": 17408,
+      "sha256_tokenizer_json": "358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394",
+      "review_docs": [
+        "provenance/tokenizer-selection/CHOSEN_CUTOFF.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "tokenizer.polytonic-plus-5120",
+      "type": "tokenizer",
+      "status": "payload-present",
+      "public_path": "tokenizer/polytonic-plus-5120",
+      "base_model": "swiss-ai/Apertus-8B-2509",
+      "vocab_size": 153600,
+      "base_vocab_size": 131072,
+      "added_modern_greek_tokens": 17408,
+      "added_polytonic_greek_tokens": 5120,
+      "sha256_tokenizer_json": "b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad",
+      "review_docs": [
+        "provenance/tokenizer-selection/CHOSEN_CUTOFF.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "dataset.cpt-7b-mix",
+      "type": "dataset",
+      "status": "hydration-pointer",
+      "public_path": "training-data/cpt-7b-mix",
+      "clariden_jsonl": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.jsonl",
+      "clariden_megatron_prefix": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document",
+      "rows": 5754172,
+      "base_tokenized_tokens": 9831704774,
+      "mix_builder_target_tokens": 7000141612,
+      "review_docs": [
+        "provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md",
+        "provenance/dataset-build/MIX_RECIPE.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "checkpoint.td-layer11-init-r17-tp2",
+      "type": "checkpoint",
+      "status": "hydration-pointer",
+      "public_path": "checkpoints/td-layer11-init-r17-tp2",
+      "format": "Megatron torch_dist TP=2, R17 patched",
+      "clariden_path": "/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release",
+      "depends_on": [
+        "tokenizer.modern-greek-17408"
+      ],
+      "review_docs": [
+        "provenance/token-distillation/RUN_LOG_20260523.md",
+        "provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "checkpoint.td-layer11-cpt-3p5b-iter834",
+      "type": "checkpoint",
+      "status": "hydration-pointer",
+      "public_path": "checkpoints/td-layer11-cpt-3p5b-iter834",
+      "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion",
+      "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834",
+      "clariden_hf_eval_copy": "/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf",
+      "depends_on": [
+        "tokenizer.modern-greek-17408",
+        "dataset.cpt-7b-mix",
+        "checkpoint.td-layer11-init-r17-tp2"
+      ],
+      "review_docs": [
+        "results/SUMMARY.md",
+        "provenance/evals/CONTINUATION_3P5B_RESULTS_20260525.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "checkpoint.baseline-vanilla-3p5b-iter834",
+      "type": "checkpoint",
+      "status": "hydration-pointer",
+      "public_path": "checkpoints/baselines/vanilla-3p5b-iter834",
+      "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion",
+      "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834",
+      "depends_on": [
+        "dataset.cpt-7b-mix"
+      ],
+      "review_docs": [
+        "results/SUMMARY.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "checkpoint.baseline-retok-3p5b-iter834",
+      "type": "checkpoint",
+      "status": "hydration-pointer",
+      "public_path": "checkpoints/baselines/retok-3p5b-iter834",
+      "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion",
+      "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834",
+      "depends_on": [
+        "tokenizer.modern-greek-17408",
+        "dataset.cpt-7b-mix"
+      ],
+      "review_docs": [
+        "results/SUMMARY.md"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    },
+    {
+      "artifact_id": "results.continuation-3p5b",
+      "type": "evaluation-summary",
+      "status": "compact-evidence",
+      "public_path": "results",
+      "review_docs": [
+        "results/SUMMARY.md",
+        "results/benchmark_table.csv"
+      ],
+      "github_code_refs": [
+        "code-links/github_subproject_manifest.json"
+      ]
+    }
+  ]
+}
+

README.md

@@ -1,88 +1,99 @@
----
-language:
-- el
-tags:
-- tokenizer
-- bpe
-- greek
-- apertus
-- tokenizer-extension
-- byte-level-bpe
-datasets:
-- fffoivos/glossapi-greek-nanochat-pretraining-dataset
-base_model:
-- swiss-ai/Apertus-8B-2509
----
-
-# Apertus Tokenizer Extension
-
-Current canonical artifact: **C3 + 17,408 curated/backfilled added units**.
-
-The active ship tokenizer is:
+# Apertus Greek Tokenizer Extension
+
+This release packages the Greek tokenizer-extension work around its main
+artifacts:
+
+1. a modern Greek merge-rule tokenizer extension for `swiss-ai/Apertus-8B-2509`;
+2. an optional polytonic/ancient Greek stacked tokenizer;
+3. the CPT dataset recipe and hydration pointers used for the bakeoff;
+4. the init and Token Distillation checkpoint pointers;
+5. compact benchmark evidence from the 2B bakeoff and 3.5B continuation.
+
+The runnable scripts live in GitHub, not in this Hugging Face artifact repo:
 
 ```text
-tokenizers/c3_added_17408_curated_padded/
+https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation
 ```
 
-It starts from `swiss-ai/Apertus-8B-2509`, preserves the Apertus base ids
-`0..131071` verbatim, adds `17,408` C3 Greek-extension BPE units, and keeps
-the final vocab aligned at `148,480 = 128 * 1160 = 256 * 580`.
+## Main Artifacts
 
-## Canonical Tokenizer
+| Artifact | Status | Path |
+|---|---|---|
+| Modern Greek tokenizer, 148,480 vocab | payload included | `tokenizer/modern-greek-17408/` |
+| Polytonic-stacked tokenizer, 153,600 vocab | payload included | `tokenizer/polytonic-plus-5120/` |
+| TD layer 11 CPT checkpoint at 3.5B | hydration pointer | `checkpoints/td-layer11-cpt-3p5b-iter834/` |
+| TD layer 11 R17-patched init | hydration pointer | `checkpoints/td-layer11-init-r17-tp2/` |
+| Vanilla/ReTok 3.5B baselines | hydration pointers | `checkpoints/baselines/` |
+| CPT 7B text mix recipe | manifest and pointer | `training-data/cpt-7b-mix/` |
+| 3.5B benchmark summary | compact evidence | `results/` |
+| Source-code links | GitHub pointers | `code-links/` |
+| Audit trail | compact provenance | `provenance/` |
 
-| Path | Vocab | Added units | Curation | SHA-256 tokenizer.json |
-| --- | ---: | ---: | --- | --- |
-| `tokenizers/c3_added_17408_curated_padded/` | 148,480 | 17,408 | 69 noisy raw-cutoff tokens structurally skipped and backfilled | `358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394` |
+## Tokenizers
 
-The cutoff can be revisited after downstream embedding adaptation and CPT,
-but this is the current handoff artifact for the Apertus Greek tokenizer
-extension line.
+Modern-only tokenizer:
 
-## Cutoff Evidence
+- base Apertus vocab: `131072`;
+- added modern Greek C3 tokens: `17408`;
+- total vocab: `148480`;
+- SHA-256 for `tokenizer.json`:
+  `358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394`.
 
-The compact evidence bundle is under:
+Polytonic-stacked tokenizer:
 
-```text
-experiments/02_1_7_intrinsic_eval_sweep_20260518/
-```
+- base Apertus vocab: `131072`;
+- modern Greek extension: `17408`;
+- polytonic/ancient Greek extension: `5120`;
+- total vocab: `153600`;
+- SHA-256 for `tokenizer.json`:
+  `b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad`.
 
-Important files:
+Both tokenizers preserve Apertus base ids and the first 1000 special/reserved
+ids.
 
-- `CHOSEN_CUTOFF.md` — pinned decision and downstream contract.
-- `REPORT.md` — intrinsic/fertility/MorphScore evidence base.
-- `artifacts/plots/` — report plots for the cutoff decision.
-- `manifests/curated_padded_at_17408_manifest.json` — exact backfilled construction.
-- `manifests/removal_mask_at_17408.jsonl` — the 69 in-cutoff tokens filtered out.
-- `firing_counts_c3_added_17408_curated_padded/` — compact token firing-count attribution artifacts for the exact C3 BPE training corpus, split by GlossAPI-nanochat, HPLT, combined C3, and exact `source_dataset`.
+## Dataset
 
-The full sweep zoo is intentionally not mirrored here. This repo keeps the
-canonical tokenizer and just enough evidence to explain why it is current.
-Raw local variants, TokEval vendor caches, parquet outputs, and large geometry
-arrays remain outside the Hub artifact set.
+The CPT text mix is built from:
 
-## Historical Artifacts
+- `fffoivos/glossapi-greek-nanochat-pretraining-dataset`;
+- nanochat internal dedup metadata;
+- Apertus-overlap drop overlay from
+  `fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z`;
+- non-Greek replay, code, and math.
 
-Older C1/C2/C3 full-tokenizer candidates and language-attribution artifacts are
-still present in their historical paths, including:
+Bulk recipe:
 
-- `continuous/`
-- `fresh/`
-- `analysis/`
-- `artifacts/language_attribution_20260515/`
+- Greek: `70%`;
+- non-Greek replay: `24%`;
+- code: `4%`;
+- math: `2%`.
 
-Those are retained for provenance, but the current tokenizer handoff is the
-17,408 curated/backfilled artifact above.
+The final text stream is about `7.0B` extended-tokenizer-budget tokens. The
+same NFC JSONL stream contains `9,831,704,774` base-tokenized Megatron tokens
+after preprocessing.
 
-## Source Code
+## Current Result Anchor
 
-Process docs and scripts live in GitHub:
+The latest compact result is the 3.5B continuation summary in `results/`.
 
-```text
-https://github.com/fffoivos/glossapi-tokenizer-extension
-```
+At iter 834:
 
-Relevant commit for this upload:
+| Arm | Greek aggregate | English retention | Multilingual | Heldout BPC, lower better |
+|---|---:|---:|---:|---:|
+| Vanilla | 0.4339 | 0.6782 | 0.4923 | 0.4724 |
+| ReTok | 0.4246 | 0.6786 | 0.4864 | 0.5390 |
+| TD layer 11 | 0.4344 | 0.6865 | 0.4967 | 0.5054 |
+
+Reading: TD layer 11 is the strongest final benchmark arm overall after the
+3.5B continuation. Vanilla remains the heldout-BPC reference.
+
+## Source And Provenance
+
+- `MANIFEST.json`: machine-readable artifact inventory.
+- `ARTIFACT_GRAPH.md`: dependency graph.
+- `code-links/github_subproject_manifest.json`: source-code pointers.
+- `provenance/`: compact documents and verification outputs for tokenizer
+  selection, dataset build, TD, conversion, and eval.
+- `archive/legacy-hf-layout-index.md`: index of older HF folders retained for
+  traceability.
 
-```text
-9a6b039 Add firing count attribution workflow
-```

SHA256SUMS

@@ -1,23 +1,59 @@
-e6e321ad419e32ce43427bba05a02a23133f05500a3486f98c5de232830c1844  ./README.md
-5c2140f0d646a2f5eaf7ea07c033fade7e28b37468eeb8595f61f6044f6cf109  ./SHA256SUMS
-f6742a090b39f1f576d18481bb1e284720275b0ab417393f219650821e83a72a  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/CHOSEN_CUTOFF.md
-3d68bbd6e6c9e25df597f46819cbe57e336b62b8a9b5b441aadeae5346e70eb7  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/README.md
-9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/REPORT.md
-60a57f77e13d8204c7728e6e5a117cb947c156c97e2c57db7de587965ada58bf  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/morphscore_greek_results.json
-7dd2ae959af9504d522cf4b606c573e6cefdd71c009fe8a7269eb051382351f0  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/curated_metrics_by_slice.csv
-76ad56410ada9ef2d7649d3babb757287993982e03fc98057e7289ec837c990c  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/extended_metrics_by_slice.csv
-190c58ce62f8878c92499ef7da9bceb26f27a1b5f25927e6681d80df59359c79  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/metrics_by_slice.csv
-f823dcc20f7cc2d7435fda6c5c09e65d704de10d37330f9ed930fdcf7f75506b  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/padded_17408_metrics.csv
-9e94927189811f9c5df2e6f75c3e4ffc4738d4305a35eef2b8499c6bf15c5acf  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/summary.json
-7bbfd6c76cc88190a2334ab45af03f77e8389830e46e8dcc4ef896f7e2e3b8fc  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/extended_4metric_to_25k.png
-0f472db11e1f6809b2369727de9d78e224d1c5916701141eae0c5207149d0681  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/fertility_per_in_domain_slice.png
-7b90ed2c76344942c48fbb445386e16fde9e0a9af0fdbd03e96054aee1ec38d3  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/headline_4metric_in_domain.png
-f72541da46b7539acca2c0fc9d641d751f0461528598e9d82651e3722681c226  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/knee_analysis.png
-4f0148ff0c94a8b67e329517ab260963daa027c0ab64ac992cffe4e95c073665  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/morphscore_greek.png
-763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/curated_padded_at_17408_manifest.json
-7874469554d319addc7f113e6b5d4b7f8984e3bd4db9b3dcaa3e4e476cf392f0  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/removal_mask_at_17408.jsonl
-0b3c2f295d1f14c9409b39ca2e6fc98346037a7d11a7cd931b59fcff7824b2e9  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/tokeval_commit.txt
-7147abc0dd32b702a399e02f47cea5576594ee6ec0d089a4e1fb6d822797aad3  ./tokenizers/c3_added_17408_curated_padded/README.md
-358c249e2fb29060c6b73157d428853b0c48710deffc8ee670ab1013880946c9  ./tokenizers/c3_added_17408_curated_padded/special_tokens_map.json
-358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394  ./tokenizers/c3_added_17408_curated_padded/tokenizer.json
-737b93393573376f6b4707ebfbb17688d0c358cba0daa3be12b4f9f57f9cb2ae  ./tokenizers/c3_added_17408_curated_padded/tokenizer_config.json
+d528c230d2d3e151776332036669794eaacc5962a1dd3b3805a64c56e2f539ff  ./ARTIFACT_GRAPH.md
+6c8fe007c045e4be44f2825d16724be5cc80d53956b1aba3d64b8843f42f2c9c  ./MANIFEST.json
+e055076dd9f8fc307d2056f5c46c9f6cd99463dfe610758a45796e98200bf3cc  ./README.md
+dc2f59def39f4d46b645608e789ebca000b3f3257336c393b234da9b9c26f607  ./archive/legacy-hf-layout-index.md
+620d1591179643c31aca8c3ba98492204fc2dd3388afff5764726d632b28fd09  ./archive/legacy_SHA256SUMS_20260525
+2eee70665c1fd320ebc5d664d5fe3b526be32adf33be71f63a0edd3b52545076  ./checkpoints/baselines/retok-3p5b-iter834/README.md
+be50de74fda67fb79a2d4ae616f3eaa0800e46b641b1ae3bd5a5d12e1fd6aeb2  ./checkpoints/baselines/retok-3p5b-iter834/hydration.md
+1a4aff5d2d4dec2424a50fb7921d3bb92fc9a92d3e6b539a4bc2b542dfefa9a5  ./checkpoints/baselines/retok-3p5b-iter834/manifest.json
+ea3e33007aa0ddfff2f88039b9f7d6a16c1317b14bee999ab762fc6867000649  ./checkpoints/baselines/vanilla-3p5b-iter834/README.md
+1531171806fc72e45ba4467a1872a40ed1360ae5b6926cd7e313f818d924e4cd  ./checkpoints/baselines/vanilla-3p5b-iter834/hydration.md
+c7203eaed58b9b9a23d21e7b25ac37dbf9ee07538707626181f06795b1ae26c1  ./checkpoints/baselines/vanilla-3p5b-iter834/manifest.json
+553be4bd24ccfc8bef210913c5e6ff7a26ca909b782bbd266ad927db07709581  ./checkpoints/td-layer11-cpt-3p5b-iter834/README.md
+6d0724888604be70abae50448eb120160710b875b42dec21ce7419a82fa60c01  ./checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md
+69b48c1c95a4c79cfdbab0ec12a669e0bc74549294e3c38dfde57d547dd2d2f6  ./checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json
+4bcb2a3a6316ee1373a170d0d4bb9e32ec25dafb942c10fb781188216faceafd  ./checkpoints/td-layer11-init-r17-tp2/README.md
+da19cd1247b0401eb6d6e25d8c0fc496ee5d93b2e745b05ce8ff59b6b9fd8c57  ./checkpoints/td-layer11-init-r17-tp2/hydration.md
+98d6a3fa9b342949b4972514925bff2952de06a202bcfcc56b449741ea4530c6  ./checkpoints/td-layer11-init-r17-tp2/manifest.json
+44a71b0a1730bf13f3387ccf15967bf036f519f002b775cd4e7a671a44c9830a  ./code-links/README.md
+44705bc53f80deba2ac02336a29a77a6fa1b83431ec10fe9b100b5493903956f  ./code-links/github_subproject_manifest.json
+06ddc69f4787f470b85d4be55992f898b98d8ab4fd8ab34a89a4d3bb96eb7e33  ./provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json
+496fa7d6f5622d6038aa2e333adbbd142ac83341e834acedc697892d66031c44  ./provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json
+61c13b8d63a692fad24e7cdf5fe5b5d6e2fb82bc4cf61981ea7c8b62b42569a6  ./provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md
+6ab3047ac06c925073ca06c1a91e91bf1c1ce4e4a1336f41f6ef35769228ebd1  ./provenance/dataset-build/MIX_RECIPE.md
+2dafea9a789b717e1f8b6fb931d8b47483e3b8433ceb1fe4b8064afa3bf5489a  ./provenance/dataset-build/bulk_mix_text_document.manifest.json
+8fbed782f3250003ef427d56ff27ae4379311da5a6263e078466ce06d6078bd2  ./provenance/evals/CONTINUATION_3P5B_RESULTS_20260525.md
+d336b0f9fa29db6ce9052ae5fcac15e55acf8f4dc0e1385c820076c43c1e4a42  ./provenance/evals/EVAL_RECIPE.md
+c10f3693691c657945530ee70bfcca8ad85bd3b42939f1ff321c9aa27b085112  ./provenance/evals/V4_BENCHMARK_COMPARISON.md
+80e2fda4e1c0ce1e2e066986add6c9e76db592f9d9b20b0b0f6857209c101d7d  ./provenance/token-distillation/RUN_LOG_20260523.md
+acda2215c1443fdd3ad25ced78085c121139640adf6d4f3037f776ec6e13f47d  ./provenance/token-distillation/TOKEN_DISTILLATION_PLAN.md
+fada1d3fa64b8060b17a65bc8b4fa8739e560454fcfe172d681ce055a51a38ba  ./provenance/tokenizer-selection/CHOSEN_CUTOFF.md
+9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25  ./provenance/tokenizer-selection/CUTOFF_SWEEP_REPORT.md
+763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895  ./provenance/tokenizer-selection/curated_padded_at_17408_manifest.json
+6046eb8f4cabda22fddf8cf5ed1581267b854221d03253e2994d304572b4f320  ./provenance/tokenizer-selection/firing_count_20260518_run_summary_augmented.json
+8fbed782f3250003ef427d56ff27ae4379311da5a6263e078466ce06d6078bd2  ./results/SUMMARY.md
+712a3c96216fe3159433b9b9d886a8cb22b51001a35a3778a7e5d6ba99909772  ./results/benchmark_table.csv
+669dc4a1c3252f994495d2fb1459145222443bcfa6f6ca1221e8ee92b602c56b  ./results/continuation_3p5b_summary.json
+13337a7976d4ca46c6c3db03014a2207ba81a42796c285410dc3f4ad26007cd9  ./results/plots/global_mmlu_full_el_subcategories_van_td.png
+6375661e9c50fd4b9f32db79a9c77375aaadbb878fab8a61019802978af3ac0e  ./results/plots/include_base_44_greek_subjects_van_td.png
+a44571c0aca767ace369a89534632a9b4618bc01391c0b20bb0f04f1c34a8830  ./results/plots/intrinsic_trajectories.png
+975e0c58604e6189f45182e07cf178bb94bead39130385aae36ed95c065ed2ad  ./results/plots/intrinsic_van_td.png
+ab072600c16be4aab4fe36ebd5b7fe3a216f1bf5ac391687733667f6231ff486  ./results/plots/trajectories.png
+375efc4b55341c0a1060fb9d7ccaa8774db66c8ca554683100449e15b1541a44  ./results/plots/trajectories_per_task.png
+ea2a02ecd33210ada4c9a5d1f7832246de07205cc4bbc5679aedb6959f93bfbe  ./results/plots/trajectories_per_task_van_td.png
+a54f94603549ddaf022297a2f691a688fb6283d793394f482531211b42dfa3a3  ./results/plots/trajectories_van_td.png
+d49ee1c80666418e6f36b649b63c9a80fb2808bb365d31e481a6c58c97053d11  ./results/plots/trajectories_van_td_gap.png
+55ce65390b6fbdc16a81f22e3d78c3a452e72192011b72289c0180a656052273  ./tokenizer/modern-greek-17408/README.md
+09c84b4086e9f800174942dcba688d8eefa8e40f3b60c20341699a6a7a11e2bf  ./tokenizer/modern-greek-17408/manifest.json
+816ec96e37c6d15e3cbc535dc146c898a7218f209fc154384f31fc1e6ad31ba5  ./tokenizer/modern-greek-17408/special_tokens_map.json
+358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394  ./tokenizer/modern-greek-17408/tokenizer.json
+ea64a17b41e1deaa7469212f413676129f33977ca3a48767f0ca68dc346df502  ./tokenizer/modern-greek-17408/tokenizer_config.json
+208ae8ea5f95c9182dc669c150aaf0c720e5937a402c72062fd83e1960978d6a  ./tokenizer/polytonic-plus-5120/README.md
+3881e1940ca013e9ec040fac3fb3a08a8f9b871e5105a0fec8165e0c6759acae  ./tokenizer/polytonic-plus-5120/manifest.json
+816ec96e37c6d15e3cbc535dc146c898a7218f209fc154384f31fc1e6ad31ba5  ./tokenizer/polytonic-plus-5120/special_tokens_map.json
+b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad  ./tokenizer/polytonic-plus-5120/tokenizer.json
+ea64a17b41e1deaa7469212f413676129f33977ca3a48767f0ca68dc346df502  ./tokenizer/polytonic-plus-5120/tokenizer_config.json
+d2f04aa396cf6558502b919b4503d248aef0ccfe08251625d75f8a9314272787  ./training-data/cpt-7b-mix/README.md
+f16d35ba906d5439709f189b1dc1253ba6e3d8518a828aac71c04fd42fefd0cf  ./training-data/cpt-7b-mix/hydration.md
+cafd9e0b15ab1d9f4f0eb7bc2939ccab29e9b6ab86ccc1f20e9a42a86b5760c0  ./training-data/cpt-7b-mix/manifest.json
+4918a44fd6c0643c984b6cba3914c53c9a3a656f313ec904e9ba7e5ed1303279  ./training-data/cpt-7b-mix/source_graph.json

archive/legacy-hf-layout-index.md

@@ -0,0 +1,27 @@
+# Legacy Hugging Face Layout Index
+
+The previous Hugging Face repo layout included useful but noisy top-level
+folders:
+
+- `analysis/`;
+- `artifacts/`;
+- `continuous/`;
+- `experiments/`;
+- `fresh/`;
+- `metadata/`;
+- `subprojects/`;
+- `tokenizers/`.
+
+Those paths are retained for traceability. The release entrypoints are now the
+curated folders at the repo root:
+
+- `tokenizer/`;
+- `checkpoints/`;
+- `training-data/`;
+- `results/`;
+- `provenance/`;
+- `code-links/`.
+
+Do not treat this file as a deletion plan. It is an index explaining the old
+layout while the new additive structure is reviewed.
+

archive/legacy_SHA256SUMS_20260525

@@ -0,0 +1,23 @@
+e6e321ad419e32ce43427bba05a02a23133f05500a3486f98c5de232830c1844  ./README.md
+5c2140f0d646a2f5eaf7ea07c033fade7e28b37468eeb8595f61f6044f6cf109  ./SHA256SUMS
+f6742a090b39f1f576d18481bb1e284720275b0ab417393f219650821e83a72a  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/CHOSEN_CUTOFF.md
+3d68bbd6e6c9e25df597f46819cbe57e336b62b8a9b5b441aadeae5346e70eb7  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/README.md
+9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/REPORT.md
+60a57f77e13d8204c7728e6e5a117cb947c156c97e2c57db7de587965ada58bf  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/morphscore_greek_results.json
+7dd2ae959af9504d522cf4b606c573e6cefdd71c009fe8a7269eb051382351f0  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/curated_metrics_by_slice.csv
+76ad56410ada9ef2d7649d3babb757287993982e03fc98057e7289ec837c990c  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/extended_metrics_by_slice.csv
+190c58ce62f8878c92499ef7da9bceb26f27a1b5f25927e6681d80df59359c79  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/metrics_by_slice.csv
+f823dcc20f7cc2d7435fda6c5c09e65d704de10d37330f9ed930fdcf7f75506b  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/padded_17408_metrics.csv
+9e94927189811f9c5df2e6f75c3e4ffc4738d4305a35eef2b8499c6bf15c5acf  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/summary.json
+7bbfd6c76cc88190a2334ab45af03f77e8389830e46e8dcc4ef896f7e2e3b8fc  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/extended_4metric_to_25k.png
+0f472db11e1f6809b2369727de9d78e224d1c5916701141eae0c5207149d0681  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/fertility_per_in_domain_slice.png
+7b90ed2c76344942c48fbb445386e16fde9e0a9af0fdbd03e96054aee1ec38d3  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/headline_4metric_in_domain.png
+f72541da46b7539acca2c0fc9d641d751f0461528598e9d82651e3722681c226  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/knee_analysis.png
+4f0148ff0c94a8b67e329517ab260963daa027c0ab64ac992cffe4e95c073665  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/morphscore_greek.png
+763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/curated_padded_at_17408_manifest.json
+7874469554d319addc7f113e6b5d4b7f8984e3bd4db9b3dcaa3e4e476cf392f0  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/removal_mask_at_17408.jsonl
+0b3c2f295d1f14c9409b39ca2e6fc98346037a7d11a7cd931b59fcff7824b2e9  ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/tokeval_commit.txt
+7147abc0dd32b702a399e02f47cea5576594ee6ec0d089a4e1fb6d822797aad3  ./tokenizers/c3_added_17408_curated_padded/README.md
+358c249e2fb29060c6b73157d428853b0c48710deffc8ee670ab1013880946c9  ./tokenizers/c3_added_17408_curated_padded/special_tokens_map.json
+358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394  ./tokenizers/c3_added_17408_curated_padded/tokenizer.json
+737b93393573376f6b4707ebfbb17688d0c358cba0daa3be12b4f9f57f9cb2ae  ./tokenizers/c3_added_17408_curated_padded/tokenizer_config.json

checkpoints/baselines/retok-3p5b-iter834/README.md

@@ -0,0 +1,12 @@
+# ReTok Baseline, 3.5B Iter 834
+
+Status: hydration pointer.
+
+Clariden checkpoint:
+
+```text
+/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834
+```
+
+This is the ReTok baseline for the modern-only 148480-vocab tokenizer.
+

checkpoints/baselines/retok-3p5b-iter834/hydration.md

@@ -0,0 +1,8 @@
+# Hydration
+
+On Clariden:
+
+```bash
+ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834
+```
+

checkpoints/baselines/retok-3p5b-iter834/manifest.json

@@ -0,0 +1,8 @@
+{
+  "artifact_id": "checkpoint.baseline-retok-3p5b-iter834",
+  "status": "hydration-pointer",
+  "tokenizer": "tokenizer.modern-greek-17408",
+  "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834",
+  "iteration": 834
+}
+

checkpoints/baselines/vanilla-3p5b-iter834/README.md

@@ -0,0 +1,13 @@
+# Vanilla Baseline, 3.5B Iter 834
+
+Status: hydration pointer.
+
+Clariden checkpoint:
+
+```text
+/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834
+```
+
+This baseline keeps the original Apertus tokenizer and remains the heldout-BPC
+reference in the 3.5B continuation.
+

checkpoints/baselines/vanilla-3p5b-iter834/hydration.md

@@ -0,0 +1,8 @@
+# Hydration
+
+On Clariden:
+
+```bash
+ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834
+```
+

checkpoints/baselines/vanilla-3p5b-iter834/manifest.json

@@ -0,0 +1,8 @@
+{
+  "artifact_id": "checkpoint.baseline-vanilla-3p5b-iter834",
+  "status": "hydration-pointer",
+  "tokenizer": "swiss-ai/Apertus-8B-2509 base tokenizer",
+  "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834",
+  "iteration": 834
+}
+

checkpoints/td-layer11-cpt-3p5b-iter834/README.md

@@ -0,0 +1,30 @@
+# TD Layer 11 CPT Checkpoint, 3.5B Iter 834
+
+Status: hydration pointer.
+
+This is the leading extended-tokenizer checkpoint after the 3.5B continuation.
+
+Clariden checkpoint:
+
+```text
+/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834
+```
+
+HF-format eval copy:
+
+```text
+/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf
+```
+
+Tokenizer:
+
+```text
+../../tokenizer/modern-greek-17408/
+```
+
+Result summary:
+
+```text
+../../results/SUMMARY.md
+```
+

checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md

@@ -0,0 +1,15 @@
+# Hydration
+
+This release entry is a pointer, not a checkpoint payload.
+
+On Clariden, use:
+
+```bash
+ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834
+ls -lh /capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf
+```
+
+Before publishing this as loadable HF weights, verify the HF eval copy contains
+the expected config, tokenizer, and safetensor shards, then upload only that
+inference-ready payload.
+

checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json

@@ -0,0 +1,11 @@
+{
+  "artifact_id": "checkpoint.td-layer11-cpt-3p5b-iter834",
+  "status": "hydration-pointer",
+  "tokenizer": "tokenizer.modern-greek-17408",
+  "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834",
+  "hf_eval_copy": "/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf",
+  "training_stage": "3.5B continuation",
+  "iteration": 834,
+  "source_code": "https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation"
+}
+

checkpoints/td-layer11-init-r17-tp2/README.md

@@ -0,0 +1,13 @@
+# TD Layer 11 Init, R17-Patched TP=2
+
+Status: hydration pointer.
+
+Clariden checkpoint:
+
+```text
+/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release
+```
+
+This is the Token Distillation layer-11 initialization in Megatron TP=2 format,
+after the R17/xIELU/QK-Norm preservation patch and roundtrip verification.
+

checkpoints/td-layer11-init-r17-tp2/hydration.md

@@ -0,0 +1,14 @@
+# Hydration
+
+On Clariden:
+
+```bash
+ls -lh /iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release
+```
+
+Verification evidence:
+
+```text
+../../provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json
+```
+

checkpoints/td-layer11-init-r17-tp2/manifest.json

@@ -0,0 +1,9 @@
+{
+  "artifact_id": "checkpoint.td-layer11-init-r17-tp2",
+  "status": "hydration-pointer",
+  "tokenizer": "tokenizer.modern-greek-17408",
+  "source_checkpoint": "/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release",
+  "format": "Megatron torch_dist TP=2, R17 patched",
+  "verification": "../../provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json"
+}
+

code-links/README.md

@@ -0,0 +1,17 @@
+# Source Code Links
+
+The runnable implementation lives in GitHub:
+
+```text
+https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation
+```
+
+This Hugging Face release keeps artifact payloads, manifests, compact evidence,
+and hydration instructions. It does not mirror the full scripts tree.
+
+See:
+
+```text
+github_subproject_manifest.json
+```
+

code-links/github_subproject_manifest.json

@@ -0,0 +1,73 @@
+{
+  "repo": "https://github.com/fffoivos/glossapi-tokenizer-extension",
+  "branch": "main",
+  "source_commit": "7c67cf11eb3d31f0ddd006f7d709709037db93a1",
+  "subproject": "subprojects/03_apertus_extension_and_embedding_adaptation",
+  "script_families": [
+    {
+      "family": "tokenizer_shipping",
+      "paths": [
+        "03_3_cscs_experiments_kickoff/ship/apertus_greek_modern_only_148480",
+        "03_3_cscs_experiments_kickoff/ship/apertus_greek_extended_153600"
+      ],
+      "artifacts": [
+        "tokenizer.modern-greek-17408",
+        "tokenizer.polytonic-plus-5120"
+      ]
+    },
+    {
+      "family": "corpus_build",
+      "paths": [
+        "03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md",
+        "03_4_implementation_experiments/init_bakeoff/corpus_build"
+      ],
+      "artifacts": [
+        "dataset.cpt-7b-mix"
+      ]
+    },
+    {
+      "family": "init_and_conversion",
+      "paths": [
+        "03_4_implementation_experiments/init_bakeoff/arms",
+        "03_4_implementation_experiments/init_bakeoff/megatron_patches"
+      ],
+      "artifacts": [
+        "checkpoint.td-layer11-init-r17-tp2",
+        "checkpoint.baseline-vanilla-3p5b-iter834",
+        "checkpoint.baseline-retok-3p5b-iter834"
+      ]
+    },
+    {
+      "family": "token_distillation",
+      "paths": [
+        "TOKEN_DISTILLATION_PLAN.md",
+        "03_4_implementation_experiments/init_bakeoff/token_distillation"
+      ],
+      "artifacts": [
+        "checkpoint.td-layer11-init-r17-tp2"
+      ]
+    },
+    {
+      "family": "training",
+      "paths": [
+        "TRAINING_RECIPE.md",
+        "03_4_implementation_experiments/init_bakeoff/bakeoff_training"
+      ],
+      "artifacts": [
+        "checkpoint.td-layer11-cpt-3p5b-iter834",
+        "checkpoint.baseline-vanilla-3p5b-iter834",
+        "checkpoint.baseline-retok-3p5b-iter834"
+      ]
+    },
+    {
+      "family": "evaluation",
+      "paths": [
+        "03_4_implementation_experiments/init_bakeoff/eval"
+      ],
+      "artifacts": [
+        "results.continuation-3p5b"
+      ]
+    }
+  ]
+}
+

provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json

@@ -0,0 +1,43 @@
+{
+  "orig_only": [],
+  "trip_only": [],
+  "standard_max_abs_diff": 0.0,
+  "r17_max_abs_diff": 0.0,
+  "xielu_max_abs_diff": 0.0,
+  "qk_norm_max_abs_diff": 0.0,
+  "standard_changed_over_tol": [],
+  "r17_changed_over_tol": [],
+  "shape_mismatches": [],
+  "standard_changed_over_tol_count": 0,
+  "r17_changed_over_tol_count": 0,
+  "logits": {
+    "logit_max_abs_diff": 0.0,
+    "logit_mean_abs_diff_max": 0.0,
+    "per_prompt": [
+      {
+        "prompt": "Η Αθήνα είναι η πρωτεύουσα της",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 72499,
+        "roundtrip_top_id": 72499,
+        "top_id_match": true
+      },
+      {
+        "prompt": "Ο Όμηρος έγραψε",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 132093,
+        "roundtrip_top_id": 132093,
+        "top_id_match": true
+      },
+      {
+        "prompt": "The capital of Greece is",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 132048,
+        "roundtrip_top_id": 132048,
+        "top_id_match": true
+      }
+    ]
+  }
+}

provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json

@@ -0,0 +1,43 @@
+{
+  "orig_only": [],
+  "trip_only": [],
+  "standard_max_abs_diff": 0.0,
+  "r17_max_abs_diff": 0.0,
+  "xielu_max_abs_diff": 0.0,
+  "qk_norm_max_abs_diff": 0.0,
+  "standard_changed_over_tol": [],
+  "r17_changed_over_tol": [],
+  "shape_mismatches": [],
+  "standard_changed_over_tol_count": 0,
+  "r17_changed_over_tol_count": 0,
+  "logits": {
+    "logit_max_abs_diff": 0.0,
+    "logit_mean_abs_diff_max": 0.0,
+    "per_prompt": [
+      {
+        "prompt": "Η Αθήνα είναι η πρωτεύουσα της",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 72499,
+        "roundtrip_top_id": 72499,
+        "top_id_match": true
+      },
+      {
+        "prompt": "Ο Όμηρος έγραψε",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 8329,
+        "roundtrip_top_id": 8329,
+        "top_id_match": true
+      },
+      {
+        "prompt": "The capital of Greece is",
+        "max_abs": 0.0,
+        "mean_abs": 0.0,
+        "reference_top_id": 48214,
+        "roundtrip_top_id": 48214,
+        "top_id_match": true
+      }
+    ]
+  }
+}

provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md

@@ -0,0 +1,108 @@
+# CPT Dataset Build Runbook
+
+This is the repeatable path for building an Apertus CPT dataset from the
+published nanochat corpus.
+
+## Inputs
+
+- Source corpus:
+  `fffoivos/glossapi-greek-nanochat-pretraining-dataset`
+- Nanochat internal dedup bundle:
+  `dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata`
+- Apertus hard-drop overlay:
+  `fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z`
+  at `artifacts/dedup_20260519T010924Z/cpt_final_overlay/apertus_overlap_drop_docs.parquet`
+
+The correct order is:
+
+1. Download the published nanochat source rows.
+2. Hard-exclude `apertus_overlap_drop_docs.parquet`.
+3. Replay nanochat internal dedup with `drop_intra_and_inter`.
+4. Build the final CPT mix/export from the remaining pool.
+
+The order matters: if Apertus-overlapping docs are removed first, an internal
+duplicate family can still keep a fresh alternate representative.
+
+## GCP Scratch VM
+
+Use a scratch VM with a large disk. A practical starting point is a
+`c4-highmem-32` or larger VM with a 1-2 TB SSD/PD disk. Build artifacts should
+be uploaded to Hugging Face or GCS before teardown.
+
+Example setup on the VM:
+
+```bash
+sudo apt-get update
+sudo apt-get install -y git python3-pip
+python3 -m pip install --user duckdb pandas pyarrow huggingface_hub typer blake3 polars
+
+git clone https://github.com/<your-org-or-user>/glossapi-tokenizer-extension.git
+cd glossapi-tokenizer-extension
+export PYTHONPATH="$PWD"
+```
+
+## Download Inputs
+
+Do not build from the old local `/home/foivos/data/glossapi_work/hf_release_publish`
+copy. Hydrate from the live HF repo so the `source_dataset` names and derived
+`doc_key` values match the Apertus overlay and the `wave2` dedup bundle.
+
+```bash
+export WORK=/mnt/disks/cpt_build
+mkdir -p "$WORK"
+
+huggingface-cli download fffoivos/glossapi-greek-nanochat-pretraining-dataset \
+  --repo-type dataset \
+  --local-dir "$WORK/nanochat" \
+  --include 'data/*.parquet' \
+  --include 'dedup_metadata/latest.json' \
+  --include 'dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata/*'
+
+huggingface-cli download fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z \
+  --repo-type dataset \
+  --local-dir "$WORK/apertus_audit" \
+  --include 'artifacts/dedup_20260519T010924Z/cpt_final_overlay/*'
+```
+
+## Build Fresh Deduped Pool
+
+```bash
+export DEDUP_ROOT="$WORK/nanochat/dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata"
+export APERTUS_DROP="$WORK/apertus_audit/artifacts/dedup_20260519T010924Z/cpt_final_overlay/apertus_overlap_drop_docs.parquet"
+export SELECTED="$WORK/cpt/selected_after_apertus_and_internal_dedup.parquet"
+
+python3 -m glossapi_corpus_cli.cli mix-prepare-selected-input \
+  --output-root "$WORK/nanochat" \
+  --selected-input-path "$SELECTED" \
+  --exclude-doc-keys-path "$APERTUS_DROP" \
+  --dedup-metadata-root "$DEDUP_ROOT" \
+  --dedup-action drop_intra_and_inter \
+  --dedup-exact-stage strict_and_relaxed \
+  --dedup-similarity-threshold 0.85 \
+  --dedup-inter-dataset-policy share_aware
+```
+
+`$SELECTED` is the deduplicated, Apertus-fresh source pool. If the CPT recipe
+uses the whole remaining pool, this is the main dataset artifact.
+
+## Optional Source Mix
+
+If the CPT run needs a sampled or weighted mix, build it from `$SELECTED`:
+
+```bash
+python3 -m glossapi_corpus_cli.cli mix-build-from-selected-input \
+  --selected-input-path "$SELECTED" \
+  --mix-output-path "$WORK/cpt/final_cpt_mix.parquet" \
+  --source-mix-config-path "$WORK/cpt/source_mix.json"
+```
+
+## Publish And Teardown
+
+Upload the final selected pool or mix before deleting the VM:
+
+```bash
+huggingface-cli upload <target-hf-dataset-repo> "$WORK/cpt" . --repo-type dataset
+```
+
+Then verify the HF repo contains the final parquet(s), summaries, and the
+`cpt_final_overlay/summary.json` provenance before tearing down the VM.

provenance/dataset-build/MIX_RECIPE.md

@@ -0,0 +1,173 @@
+# Mix recipes — bulk and anneal
+
+The bakeoff (and future production CPT) consumes a stream of documents
+interleaved from several sources with explicit per-source weights. Per
+[`../../cpt_plan.md`](../../cpt_plan.md) v0.7 §2 + §4:
+
+- **Bulk phase**: one shuffled-mixture stream, all corpora at target weights from token 0. This is what the bakeoff actually trains on.
+- **Anneal phase**: final 10–20 % of *production* training (out of scope for the bakeoff). Mixture shifts to highest-priority subsets; replay drops from ~30 % → ~15 %; LR decays. We define the recipe here for completeness; the bakeoff never executes it.
+
+Both recipes are JSON, in `recipes/`. The mix builder (`mix_builder.py`) reads either and emits a JSON-lines stream.
+
+## Three-step build path (reviewer round-2 Blocker 3 fix)
+
+The CPT dataset build follows the canonical runbook at [`../../../03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md`](../../../03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md). **Order matters**: if Apertus-overlapping docs are removed first, an internal-duplicate family can still keep a fresh alternate representative.
+
+```
+1. bash pull_greek_corpus.sh        # nanochat parquets + dedup_metadata + Apertus overlay
+2. bash prepare_greek_pool.sh       # → $SELECTED parquet (Apertus-drop + drop_intra_and_inter)
+3. SELECTED=$WORK/cpt/selected_after_apertus_and_internal_dedup.parquet \
+   python3 mix_builder.py --recipe recipes/bulk.json --target-tokens 7_000_000_000 ...
+```
+
+All six Greek `bulk.json` sources point at `local_parquet: ${SELECTED}` — the single post-dedup parquet — and filter by `source_dataset` value to slice into HPLT / literary / dialogue / academic / legal / dictionary buckets. The Apertus-overlap-drop is applied **upstream** in step 2 (not per-source in mix_builder), so it applies uniformly to all six categories rather than only to HPLT (the previous bug). Internal dedup with `drop_intra_and_inter` also happens in step 2 (was missing entirely before).
+
+## Composition (bulk recipe — what the bakeoff actually consumes)
+
+Working defaults per v0.7 §2 + §4 + user answers 2026-05-20:
+
+| Top-level bucket | Share | Sub-allocation |
+|---|---:|---|
+| Greek (post-Apertus-overlap-drop, post-internal-dedup) | **70 %** | see below |
+| Non-Greek replay (24 languages, 3 tiers) | **24 %** | see below |
+| Code | **4 %** | Intended: `bigcode/starcoderdata` / The Stack lineage (Apertus used StarCoder); 2026-05-21 CSCS fallback: `codeparrot/codeparrot-clean-train` |
+| Math | **2 %** | `HuggingFaceTB/finemath` config `finemath-3plus` (Apertus stage-1 source per `submit_apertus_8b.sh:L29`) |
+
+> **2026-05-21 rebalance.** Replay reduced from 26 % → 24 % to free 2 % for FineMath, taken entirely from English (FW-Edu) which went from 3.9 % → 1.9 %. The other 23 replay-language weights stay at their pre-rebalance values. Greek and code shares unchanged.
+
+> **2026-05-21 CSCS run detail.** The live full-build recipe reads replay and FineMath from staged local single-shard parquets under `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/{replay,math}`. This avoids repeated HF dataset-metadata resolution from seven concurrent shards, which hit HF's API rate limit. The Persian replay bucket uses the local FineWeb-2 `fas_Arab` fallback shard while preserving the recipe label `replay_t2_pes_Arab`.
+
+> **OPUS Greek-English parallel** is listed as optional in v0.7 §4.4 but **not yet in the bakeoff bulk mix** — it needs different schema handling in `mix_builder.py` (its rows are `{translation: {el: ..., en: ...}}` rather than `{text: ...}`). Deferred to a future iteration of corpus_build.
+
+### Greek sub-allocation (70 % of total)
+
+| Source | Share of Greek | HF id / filter |
+|---|---:|---|
+| **HPLT clean60 (fresh; Apertus-overlap overlay applied)** | 50 % of Greek = 35 % total | nanochat dataset, `source_dataset starts with HPLT__ell_Grek` + `apertus_overlap_drop_docs.parquet` exclude |
+| Literary nanochat (artos-zois + Project_Gutenberg + Ekklisiastika + 1000_prwta + klasikh + Wikisource + dimodis_logotexnia + archetai) | 26 % of Greek = 18.2 % total | nanochat dataset, filter `source_dataset` ∈ literary list |
+| Dialogue + textbooks (OPUS OpenSubtitles-el + openbook + Sxolika_vivlia + ert-press + istorima) | 9 % of Greek = 6.3 % total | nanochat dataset, filter |
+| Academic (openarchives.gr + greek_phd + Apothetirio_Kallipos + Apothetirio_Pergamos) | 8 % of Greek = 5.6 % total | nanochat dataset, filter |
+| Legal + civic (eurlex-greek-legislation + AI-team-UoA/greek_legal_code + opengov.gr-diaboyleuseis + ellinika_dedomena_europaikou_koinovouliou) | 5 % of Greek = 3.5 % total | nanochat dataset, filter |
+| Dictionary + misc (modern-greek-dictionary capped + finewiki Greek half-weight + 95k_deigma + others) | 2 % of Greek = 1.4 % total | nanochat dataset, filter; modern-greek-dictionary explicitly capped per Claude-review note in `../../collegues_Apertus_plan.md` |
+
+The HPLT half dominates by token mass (the underlying HPLT clean60 slice is ~50 B chars at v0.7 chars/token = ~14 B tokens; the GlossAPI sub-pools combined are smaller). The recipe weights are **probabilities for the interleaver**, which is what affects the per-step mix; effective token shares track these weights up to per-source exhaustion.
+
+### Non-Greek replay sub-allocation (24 % of total, across 24 languages)
+
+Tier weights per v0.7 §4.2 (40–50 % T1 / 35–45 % T2 / 10–15 % T3); working defaults at the midpoint, scaled to the **24 % outer share** (post-2026-05-21 rebalance: English's share absorbed the 2 % shift to FineMath):
+
+| Tier | Languages | Share of replay | Total share |
+|---|---|---:|---:|
+| **T1** (8 langs, FW2-HQ where Apertus used HQ) | eng, fra, deu, ita, spa, rus, arb, cmn | ~46 % | 11.0 % |
+| **T2** (11 langs, FW2 where Apertus used standard) | tur, bul, srp, ron, heb, por, pol, nld, pes, ukr, jpn | ~41 % | 9.88 % |
+| **T3** (5 langs, FW2; small per-lang) | lat, hye, kat, sqi (or als), mkd | ~13 % | 3.12 % |
+
+Within each tier, weights are roughly equal *except* English gets ~17 % of T1 (≈ 1.9 % of total, post-rebalance) — still the largest single non-Greek language, but trimmed from 3.9 % to 1.9 % to make room for FineMath at 2 %. The remaining 7 T1 langs each stay at 1.3 % of total.
+
+Per-language sources:
+
+- T1 (Apertus quality-filtered the 20 high-resource langs at top-10 % per-language XLM-R): **`epfml/FineWeb2-HQ`** with `config_name = <iso639-3>_<script>` (Messmer, Sabolčec, Jaggi 2025, [arXiv:2502.10361](https://arxiv.org/abs/2502.10361); top-10 %, NOT "Score-3" — that's FineWeb-Edu's filter).
+- T2 / T3 (Apertus used unfiltered FW2 for these): **`HuggingFaceFW/fineweb-2`** with `config_name = <iso639-3>_<script>` (v2.0.1 per Apertus tech report footnote 18).
+- English specifically: prefer **`HuggingFaceFW/fineweb-edu`** Score-3 (Penedo et al. [arXiv:2406.17557](https://arxiv.org/abs/2406.17557); matches Apertus stage-5 cooldown). FineWeb2-HQ English is the fallback.
+
+> **Reviewer flag**: cpt_plan.md v0.7 §4.2's "24 replay languages" framing (T1+T2+T3 = 8+11+5) is **our internal grouping**, not Apertus's. Apertus's tech report (arXiv:2509.14233 Appendix G, p.88-89) enumerates **20 high-resource languages** that receive quality+toxicity filtering; the other ~4 languages we include (T3: lat/hye/kat/sqi/mkd) are added by us for cultural/regional coverage relevant to Greek. Token-share targets per language are also our derivation, not from Apertus (the tech report only publishes document counts and percentages of FineWeb-2 documents, not training-mixture token shares).
+
+### Code (4 %)
+
+| Source | HF id |
+|---|---|
+| Intended code replay | `bigcode/starcoderdata` (Apertus's pretraining source) |
+| 2026-05-21 CSCS fallback | `codeparrot/codeparrot-clean-train` |
+
+Use the default "all-permissive" subset when StarCoder/The Stack is accessible; no language filter (mixed-language source-code is fine for retention). The 2026-05-21 CSCS run uses `codeparrot/codeparrot-clean-train` because BigCode sources were gated or script-backed under the current auth/runtime. This preserves the intended 4 % code-retention pressure, but it is not an exact StarCoder-source match.
+
+### Math (2 %)
+
+| Source | HF id + config |
+|---|---|
+| Math (web-derived high-quality math text) | `HuggingFaceTB/finemath` config `finemath-3plus` (Apertus stage-1 source per `submit_apertus_8b.sh:L29`) |
+
+FineMath-3plus is the higher-quality subset of FineMath, selected for math content (problem statements, solutions, derivations) from CommonCrawl. Apertus uses `finemath-3plus-merge` in pretraining stage 1; the bakeoff includes it at the same 2 % share Apertus's stage-1 allocates within the broader mix (Apertus stage proportions aren't published per-source — 2 % is a reasonable default).
+
+## Composition (anneal recipe — defined but not run in the bakeoff)
+
+Per v0.7 §3.2: in the final 10–20 % of production training, mixture shifts to highest-priority subsets and replay drops to ~15 %. Working pattern (between Llama 3 "narrow high-quality" and OLMo Dolmino "broad quality-curated"):
+
+| Bucket | Bulk share | Anneal share | Reason |
+|---|---:|---:|---|
+| Greek high-quality literary (artos-zois + Project_Gutenberg + Wikisource + 1000_prwta) | 12 % | **30 %** | clean prose anchor |
+| Greek academic + legal (openarchives + greek_phd + eurlex + greek_legal_code) | 9 % | **22 %** | domain depth |
+| Greek dictionary (modern-greek-dictionary uncapped from anneal start) | 1 % | **15 %** | gap closure (Apertus_plan §"Final mix") |
+| Greek dialogue + textbooks (OpenSubtitles + openbook + Sxolika + ert) | 6 % | **8 %** | maintained but de-emphasized |
+| HPLT broad (kept reduced for register breadth) | 35 % | **10 %** | safety net against narrow-corpus overfit |
+| Non-Greek replay (Tier 1 dominant) | 26 % | **12 %** | reduced per v0.7 §3.2 "30 % → 15 %" pattern; Tier 1 ≥ 70 % within replay during anneal |
+| Code | 4 % | **3 %** | maintained at slightly reduced share |
+
+The anneal recipe is `recipes/anneal.json`. It will be used in the production CPT (15–20 B tokens; final 10–20 % = ~2–4 B tokens of anneal) **after** the bakeoff picks an init winner. Not part of the bakeoff.
+
+## Builder usage
+
+```bash
+# Generate the bakeoff bulk JSON-lines stream after the smoke passes
+sbatch corpus_build/mix_builder_full.sbatch
+
+# After shard concatenation, enforce NFC on the final JSONL stream too.
+# This is CPU-only and preserves row order; unchanged rows stay byte-identical.
+sbatch corpus_build/normalize_jsonl_nfc.sbatch
+
+# Same shape for the anneal recipe (not run in the bakeoff)
+python3 mix_builder.py \
+    --recipe recipes/anneal.json \
+    --target-tokens 3_000_000_000 \
+    --tokenizer /iopsstor/scratch/cscs/fffoivos/tokenizers/apertus_greek_modern_only_148480 \
+    --output /iopsstor/scratch/cscs/fffoivos/cpt_corpus/anneal_mix.jsonl \
+    --seed 20260520
+```
+
+For sharded full builds, `mix_builder_full.sbatch` passes `--source-shard-index`
+and `--source-shard-count` into `mix_builder.py`. This partitions each source's
+eligible rows across array tasks before token-fair sampling, so the concatenated
+bulk mix is not just seven repeats of the same source prefixes.
+
+Output is JSON-lines. Each line is `{"text": "...", "source": "...", "doc_id": "...", "lang": "..."}`. Megatron-LM-Swiss-AI's `tools/preprocess_data.py` then converts JSON-lines → binary indexed dataset (`.bin` + `.idx`) for training.
+
+NFC is enforced twice: first on source parquets with `normalize_nfc.sbatch`,
+then on the final JSONL stream with `normalize_jsonl_nfc.sbatch`. The second
+pass caught a small code-replay leak in the live bakeoff mix (`37` non-NFC rows
+out of `5,754,172`) and produced
+`/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.jsonl` with manifest
+`bulk_mix.nfc.manifest.json`.
+
+Production-safe Vanilla/base-tokenizer binary, built from the NFC stream:
+
+- prefix:
+  `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document`
+- job: `2367579` on `xfer`, `COMPLETED`, `0:0`, elapsed `00:16:07`
+- rows/sequences: `5,754,172`
+- base-tokenized tokens: `9,831,704,774`
+- local evidence:
+  `production_base_nfc_preprocess_2367579/`
+
+The older bakeoff base/ext binaries were built before this NFC cleanup. The
+NFC delta is tiny and does not change the bakeoff decision, but production
+Vanilla CPT should use the `bulk_mix_base_nfc_megatron` prefix above.
+
+Determinism: the `--seed` controls the interleave randomization. Same seed means the same JSONL text stream across runs. Vanilla and the extended arms then use different tokenizers/preprocessed Megatron binaries, so token IDs differ across tokenizer families even though the document order is shared.
+
+## Why two stages (build JSONL → preprocess to .bin/.idx)
+
+Megatron's training reads its native binary format; converting on the fly during interleaving is awkward. Splitting the pipeline:
+
+1. `mix_builder.py` does the streaming interleave + budget-cap + writes JSONL (CPU job on `xfer`; do not use `normal` for CPU-only work because it allocates GPU nodes on Clariden).
+2. `normalize_jsonl_nfc.py` enforces NFC on the final JSONL stream (CPU job on `xfer`).
+3. `tools/preprocess_data.py` (Megatron) does the tokenization + binary packing (CPU job on `xfer`).
+4. Training reads the binary on `normal`.
+
+This way each stage has a single clear job, and we can re-run any stage independently.
+
+On current `xfer` nodes, `uenv`/torch are not visible, so Megatron's stock
+`tools/preprocess_data.py` may not be runnable there. For the production Vanilla
+NFC build, `preprocess_hf_jsonl_to_megatron.py` was used instead. It avoids
+torch, writes the same Megatron indexed-dataset format, and was validated
+byte-for-byte against the stock Megatron output on the first `1000` rows before
+the full job was launched.

provenance/dataset-build/bulk_mix_text_document.manifest.json

@@ -0,0 +1,19 @@
+{
+  "input": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.jsonl",
+  "output_prefix": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document",
+  "bin": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document.bin",
+  "idx": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document.idx",
+  "tokenizer_model": "/iopsstor/scratch/cscs/fffoivos/models/apertus-8b-2509",
+  "json_key": "text",
+  "append_eod": true,
+  "workers": 64,
+  "chunk_size": 64,
+  "limit": 0,
+  "rows": 5754172,
+  "sequences": 5754172,
+  "documents": 5754173,
+  "tokens": 9831704774,
+  "bytes_read": 27125565083,
+  "wall_seconds": 960.677738904953,
+  "format": "Megatron indexed dataset v1, int32 document-level"
+}

provenance/evals/CONTINUATION_3P5B_RESULTS_20260525.md

@@ -0,0 +1,91 @@
+# 3.5B continuation results - Vanilla vs ReTok vs TD layer11
+
+Generated UTC: `2026-05-25T05:23:53+00:00`.
+
+This summarizes the continuation run `continuation_3p5b_20260524T143012Z`,
+which extended Vanilla, ReTok, and TD layer11 from iter 476 (~2.0B tokens)
+to iter 834 (~3.5B tokens). Local JSON snapshots live under
+`per_iter_results/`; remote full artifacts remain on Clariden under
+`/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_*`.
+
+## Bottom line
+
+- TD layer11 is the best final benchmark arm overall: it is first on English
+  retention and multilingual aggregates, and narrowly first on the Greek
+  aggregate at iter 834.
+- Vanilla still has the best tokenizer-fair heldout Greek BPC, but its
+  downstream Greek aggregate declined during the 2.0B -> 3.5B continuation.
+- ReTok improves fastest on BPC and wins Greek MMLU / INCLUDE-44 Greek at
+  iter 834, but it remains behind TD and Vanilla on the Greek aggregate.
+- If selecting for the actual downstream bakeoff objective, TD layer11 is now
+  the leading candidate. If selecting only for heldout BPC, Vanilla remains
+  ahead.
+
+## Aggregate scoreboard at iter 834
+
+| Arm | Greek agg | Delta vs 476 | EN retention | Delta vs 476 | Multilingual | Delta vs 476 | BPC lower better | BPC delta |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| Vanilla | 0.4339 | -0.70 pp | 0.6782 | -0.36 pp | 0.4923 | +0.22 pp | 0.4724 | -0.0182 |
+| ReTok | 0.4246 | +0.96 pp | 0.6786 | +0.36 pp | 0.4864 | -0.09 pp | 0.5390 | -0.0349 |
+| TD layer11 | 0.4344 | +0.90 pp | 0.6865 | +0.38 pp | 0.4967 | +0.68 pp | 0.5054 | -0.0256 |
+
+## Per-task winners at iter 834
+
+| Group | Task | Vanilla | ReTok | TD layer11 | Winner |
+|---|---|---:|---:|---:|---|
+| EN retention | `mmlu` | 0.5330 | 0.5565 | 0.5556 | ReTok |
+| EN retention | `hellaswag` | 0.7573 | 0.7556 | 0.7632 | TD layer11 |
+| EN retention | `arc_easy` | 0.7866 | 0.7576 | 0.7837 | Vanilla |
+| EN retention | `arc_challenge` | 0.5230 | 0.5247 | 0.5358 | TD layer11 |
+| EN retention | `piqa` | 0.7927 | 0.7851 | 0.7894 | Vanilla |
+| EN retention | `winogrande` | 0.6764 | 0.6922 | 0.6914 | ReTok |
+| Multilingual | `global_mmlu` | 0.4505 | 0.4603 | 0.4628 | TD layer11 |
+| Multilingual | `xcopa` | 0.6178 | 0.5920 | 0.6156 | Vanilla |
+| Multilingual | `xnli` | 0.4087 | 0.4070 | 0.4117 | TD layer11 |
+| Greek | `global_mmlu_full_el` | 0.4145 | 0.4153 | 0.4036 | ReTok |
+| Greek | `include_base_44_greek_few_shot_en` | 0.4022 | 0.4058 | 0.4004 | ReTok |
+| Greek | `belebele_ell_Grek` | 0.5078 | 0.4933 | 0.5389 | TD layer11 |
+| Greek | `arc_challenge_mt_el` | 0.4130 | 0.3899 | 0.4061 | Vanilla |
+| Greek | `xnli_el` | 0.3831 | 0.3695 | 0.3851 | TD layer11 |
+| Greek | `xquad_el` | 0.2868 | 0.3084 | 0.3364 | TD layer11 |
+| Greek | `global_piqa_completions_ell_grek` | 0.6300 | 0.5900 | 0.5700 | Vanilla |
+
+## Change from iter 476 to iter 834
+
+| Group | Task | Vanilla delta | ReTok delta | TD layer11 delta |
+|---|---|---:|---:|---:|
+| EN retention | `mmlu` | -0.09 pp | +0.24 pp | +0.56 pp |
+| EN retention | `hellaswag` | -0.21 pp | +0.69 pp | +0.27 pp |
+| EN retention | `arc_easy` | +0.13 pp | -0.59 pp | +0.29 pp |
+| EN retention | `arc_challenge` | -0.43 pp | +0.94 pp | +0.85 pp |
+| EN retention | `piqa` | -0.22 pp | -0.11 pp | -0.27 pp |
+| EN retention | `winogrande` | -1.34 pp | +1.03 pp | +0.55 pp |
+| Multilingual | `global_mmlu` | -0.64 pp | +0.14 pp | +1.34 pp |
+| Multilingual | `xcopa` | +1.09 pp | -0.71 pp | +0.42 pp |
+| Multilingual | `xnli` | +0.22 pp | +0.30 pp | +0.28 pp |
+| Greek | `global_mmlu_full_el` | -0.69 pp | +1.62 pp | +1.77 pp |
+| Greek | `include_base_44_greek_few_shot_en` | -1.63 pp | +3.26 pp | -0.36 pp |
+| Greek | `belebele_ell_Grek` | -0.56 pp | -0.33 pp | +1.11 pp |
+| Greek | `arc_challenge_mt_el` | -0.77 pp | +1.79 pp | +0.26 pp |
+| Greek | `xnli_el` | -1.89 pp | -0.56 pp | +0.48 pp |
+| Greek | `xquad_el` | -2.33 pp | -0.08 pp | +1.02 pp |
+| Greek | `global_piqa_completions_ell_grek` | +3.00 pp | +1.00 pp | +2.00 pp |
+
+## New-token diagnostics
+
+Measured on the 500-document heldout slice. Top-k is the fraction of
+positions whose correct new token appears in the model's top-k predictions;
+lower mean rank is better.
+
+| Arm | Top-1 at new target | Delta from 585 | Top-5 at new target | Delta from 585 | Mean rank | Delta from 585 |
+|---|---:|---:|---:|---:|---:|---:|
+| ReTok | 0.3799 | +2.00 pp | 0.5469 | +2.11 pp | 228.9 | -31.4 |
+| TD layer11 | 0.4105 | +1.43 pp | 0.5811 | +1.54 pp | 174.3 | -18.8 |
+
+## Artifact checklist
+
+- Local packed-eval snapshots: `per_iter_results/{vanilla,retok,td}_iter{585,715,834}.json`.
+- Local BPC snapshots: `per_iter_results/intrinsic/*_iter{585,715,834}_fair.json`.
+- Local new-token diagnostics: `per_iter_results/diagnostics/{retok,td}_iter{585,715,834}_new_token_diagnostics.json`.
+- Regenerated plots are written to `plots/`.
+- Final remote packed eval job: `2376082`, state `COMPLETED`, exit `0:0`, elapsed `00:59:51`.

provenance/evals/EVAL_RECIPE.md

@@ -0,0 +1,321 @@
+# Eval recipe — V4 baseline + per-arm bakeoff eval
+
+What we run, when, and what to expect out. **Engine**: [`swiss-ai/lm-evaluation-harness`](https://github.com/swiss-ai/lm-evaluation-harness) (Apertus team's fork — cited in Apertus tech report [arXiv:2509.14233](https://arxiv.org/abs/2509.14233) §5.1 footnote 45). **Citation**: Gao, Tow, Abbasi et al., Zenodo 2024, DOI [10.5281/zenodo.12608602](https://doi.org/10.5281/zenodo.12608602).
+
+Two scopes:
+
+- **V4 baseline** — full eval suite on unmodified `swiss-ai/Apertus-8B-2509`. One run, before the bakeoff fires. Output: per-benchmark mean + bootstrap CI. Gates the §5.6 hard-gate thresholds (how much regression on each benchmark counts as a "failure" for an arm).
+- **Per-arm bakeoff eval** — same suite run on each arm's saved checkpoints in the 80–100 % range of its 2 B-token budget. Output: windowed-average score + bootstrap CI per arm. Drives §5.6 selection.
+
+Both scopes use the **same task list and same harness flags** so the V4 baseline is the apples-to-apples comparator.
+
+> **2026-05-21 takeover note.** V4 job `2334245` completed the then-current script, but that script accidentally omitted `global_mmlu` from Group 1. Treat its copied artifacts as a valid partial baseline for the listed tasks, not as the final Table-14 baseline. `run_eval.sbatch` now includes `global_mmlu`; rerun V4-HF and V4-post-conversion before filling final §5.6 thresholds.
+
+## Scope of tasks: match Apertus's pretraining-eval table, not post-training
+
+The bakeoff compares **pretraining-stage** checkpoints (no SFT, no DPO). Apertus's tech report reports two distinct eval tables:
+
+- **Table 14, p.38 (pretraining benchmarks)**: ARC (challenge + easy), HellaSwag, WinoGrande, PIQA, MMLU, Global-MMLU, XNLI (15 langs incl. Greek), XCOPA (11 langs). All run with harness-default shot counts per §5.1.
+- **Table 22, p.44 (post-training benchmarks)**: GSM8K-CoT, HumanEval, MBPP, IFEval, BBH, BBQ, ToxiGen, HarmBench, etc. — generative / instruction-following tasks that require an SFT'd model.
+
+Our **primary** retention metric is Table-14 tasks (Group 1 below). Table-22 tasks (GSM8K / HumanEval / MBPP / IFEval) we mark as **post-training only** and skip in the bakeoff — they shouldn't fluctuate from a CPT init choice on a non-instruction-tuned base.
+
+## Task list
+
+Three groups. The wrappers in this directory take `--task-group {full|greek_only|retention_only}` so you can run them independently if needed.
+
+### Group 1: retention (Apertus pretraining-eval set per Table 14)
+
+| Task | Capability | `lm-eval-harness` task name | Few-shot (harness default) |
+|---|---|---|---|
+| ARC-Challenge | reasoning | `arc_challenge` | 25 |
+| ARC-Easy | reasoning | `arc_easy` | 25 |
+| HellaSwag | commonsense | `hellaswag` | 10 |
+| WinoGrande | coreference | `winogrande` | 5 |
+| PIQA | physical commonsense | `piqa` | 0 |
+| MMLU (57 subjects) | knowledge | `mmlu` | 5 |
+| Global-MMLU (15 langs incl. Greek) | multilingual knowledge | `global_mmlu` | 5 |
+| XNLI (15 langs incl. Greek) | multilingual NLI | `xnli` | 0 |
+| XCOPA (11 langs) | multilingual commonsense | `xcopa` | 0 |
+
+XNLI and Global-MMLU both include Greek (`xnli_el`, `global_mmlu_el`); these double as Greek-retention signals for the bakeoff. Per Apertus §5.1 footnote 45: *"All of our reported pretraining benchmarks follow the default configuration specified in lm-evaluation-harness"* — we adopt the same convention. Tasks **excluded** from the bakeoff because Apertus reports them only post-training: GSM8K, HumanEval, MBPP, IFEval, BBH (Table 22).
+
+### Group 2: Greek (native swissai-harness tasks + Belebele Greek)
+
+**2026-05-21 update — empirically confirmed task availability.** The previous version of this table listed `arc_greek` / `hellaswag_greek` / `winogrande_greek` / `mmlu_greek` / `mmlu_pro_greek` / `truthfulqa_greek` / `medical_mcqa_greek` as `lm-eval-harness` task names. **Verified against the `swiss-ai/lm-evaluation-harness` clone on Clariden (2026-05-21, V4 attempt jobs 2333668 / 2333723):** those task names do not exist in the swissai harness — they live in the Meltemi/Krikri team's forks (`LeonVouk/lighteval`, `ilsp/lm-evaluation-harness-greek`) and have not landed upstream. The "reviewer flag" below was correct.
+
+**What we actually ship at V4** (the names below resolve in `swiss-ai/lm-evaluation-harness` and were used for V4 job 2334245):
+
+| Task | Source | `lm-eval-harness` task name | Few-shot |
+|---|---|---|---|
+| Greek ARC (machine-translated) | swissai harness | `arc_challenge_mt_el` | 25 |
+| Greek MMLU (Global-MMLU, full Greek slice) | swissai harness | `global_mmlu_full_el` | 5 |
+| Greek MMLU-44 (INCLUDE-44, native Greek, 7 subjects) | `CohereForAI/include-base-44`, config `Greek` | `include_base_44_greek_few_shot_en` (group of 7 subtasks) | 5 |
+| Greek NLI | swissai harness | `xnli_el` | 0 |
+| Greek QA | swissai harness | `xquad_el` | 0 |
+| Greek PIQA (machine-translated) | swissai harness | `global_piqa_completions_ell_grek` | 0 |
+| Belebele Greek | `facebook/belebele`, config `ell_Grek` | `belebele_ell_Grek` | 5 |
+
+> **Reviewer flag (still open) — PF5 ILSP YAMLs.** The ILSP tasks the previous table claimed (`hellaswag_greek`, `winogrande_greek`, `mmlu_pro_greek`, `truthfulqa_greek`, `medical_mcqa_greek`) genuinely add Greek signal we don't currently have. The PF5 follow-up is to **port the YAML configs from `LeonVouk/lighteval` (or `ilsp/lm-evaluation-harness-greek`) into the swissai harness clone** so they resolve. Tracked as Task #55 in our running task list. Until PF5 lands, the bakeoff runs with the seven-task Greek list above + the multilingual coverage from Group 1.
+
+Greek post-training-only tasks (`ilsp/mgsm_greek`, `ilsp/ifeval_greek`, `ilsp/mt-bench-greek`, `ilsp/m-ArenaHard_greek`) are out of scope for the pretraining-stage bakeoff — see "Scope of tasks" above.
+
+The custom Greek evals from v0.7 §6.2 — polytonic continuation (out of bakeoff scope per the modern-only directive), morphology minimal pairs, language-ID drift, register preservation — run via Inspect AI as a separate sbatch (`run_inspect_evals.sbatch`, not in this iteration).
+
+### Group 3: safety / other
+
+| Task | Source | Notes |
+|---|---|---|
+| HarmBench | `swiss-ai/harmbench` | Apertus team's curated harmful-behavior bench |
+| PolyglotToxicityPrompts | `swiss-ai/polyglotoxicityprompts` | Multilingual toxicity (incl. Greek) |
+| AttaQ Greek | `ilsp/attaq_greek` | Greek-targeted safety |
+| Jailbreak-StrongReject Greek | `ilsp/Jailbreak-StrongReject-el` | Jailbreak resistance |
+
+These are inspect-style evals, run via `inspect eval` rather than lm-eval-harness. Lower priority for the bakeoff (no §5.6 hard gate ties to them directly); useful for the eventual release.
+
+## Eval cadence
+
+Per v0.7 §6.1:
+
+- **Trajectory metrics** (training loss, per-bucket PPL, BPC trajectory, §5.3 new-token diagnostic suite): every 100 M tokens in the bakeoff (every ~25 global steps at 4 M tokens/step).
+- **Downstream benchmarks** (Group 1 + Group 2): every 500 M tokens during training, plus a full sweep at the last 3–5 checkpoints in the 80–100 % budget range for selection.
+- **V4 baseline**: once, before the bakeoff. Full suite × 1 run. Bootstrap CIs over eval samples (not over runs — most benchmark items are deterministic).
+
+For 2 B-token bakeoff per arm: that's 4 mid-training benchmark runs (at 500 M, 1.0 B, 1.5 B, 2.0 B) and the last 3 (1.0 / 1.5 / 2.0 B) feed the windowed selection score.
+
+### Current 2026-05-22 operational cadence
+
+Training writes Megatron `torch_dist` checkpoints, while lm-eval and the intrinsic metric jobs require HF-format model directories. Each evaluated checkpoint therefore has two stages:
+
+1. Convert the Megatron checkpoint to HF with [`convert_bakeoff_checkpoint_to_hf.sbatch`](convert_bakeoff_checkpoint_to_hf.sbatch).
+2. Run [`run_eval.sbatch`](run_eval.sbatch) and, when a held-out JSONL is available, [`run_tokenizer_fair_metrics.sbatch`](run_tokenizer_fair_metrics.sbatch) + [`run_new_token_diagnostics.sbatch`](run_new_token_diagnostics.sbatch).
+
+Use [`submit_bakeoff_checkpoint_eval.sh`](submit_bakeoff_checkpoint_eval.sh) for the conversion/eval chain:
+
+```bash
+RUN_TAG=bakeoff_1node_chain_20260522_005620 \
+  bash submit_bakeoff_checkpoint_eval.sh vanilla 65 greek_only
+```
+
+For full bakeoff checkpoints where all three arms should be evaluated, prefer
+the packed path. Clariden's normal partition can allocate/bill a whole 4-GPU
+node even for a one-GPU `lm-eval` job; the packed submitter converts each arm
+separately, then runs the three single-GPU full evals concurrently inside one
+node allocation:
+
+```bash
+RUN_TAG=bakeoff_1node_chain_20260522_005620 \
+SUBMIT_INTRINSIC=1 \
+EVAL_JSONL=/iopsstor/scratch/cscs/fffoivos/cpt_corpus/heldout/cpt_greek_heldout_500_20260522.jsonl \
+  bash submit_bakeoff_checkpoint_eval_packed.sh 390 full vanilla retok centroid
+```
+
+For a live run, use [`watch_and_submit_checkpoint_evals.sh`](watch_and_submit_checkpoint_evals.sh)
+to submit once per arm as soon as each checkpoint appears:
+
+```bash
+RUN_TAG=bakeoff_1node_chain_20260522_005620 \
+ITER=65 \
+TASK_GROUP=greek_only \
+POLL_SECONDS=300 \
+nohup bash watch_and_submit_checkpoint_evals.sh \
+  > /capstor/scratch/cscs/fffoivos/runs/eval/watch_iter65.log 2>&1 &
+```
+
+For full checkpoints in the live bakeoff, use
+[`watch_and_submit_checkpoint_evals_packed.sh`](watch_and_submit_checkpoint_evals_packed.sh)
+instead, so the watcher waits until all requested arms are complete and then
+submits a single packed full-eval job:
+
+```bash
+RUN_TAG=bakeoff_1node_chain_20260522_005620 \
+ITER=390 \
+TASK_GROUP=full \
+SUBMIT_INTRINSIC=1 \
+EVAL_JSONL=/iopsstor/scratch/cscs/fffoivos/cpt_corpus/heldout/cpt_greek_heldout_500_20260522.jsonl \
+POLL_SECONDS=300 \
+nohup bash watch_and_submit_checkpoint_evals_packed.sh \
+  > /capstor/scratch/cscs/fffoivos/runs/eval/watch_iter390_packed.log 2>&1 &
+```
+
+The live bakeoff saves every 65 iterations, which is about 273 M tokens (`65 × 1024 × 4096`). The practical cadence is:
+
+| Iteration | Tokens/arm | Eval action |
+|---:|---:|---|
+| 65 | ~273 M | Greek-only downstream smoke on all three arms; proves checkpoint save, conversion, and eval before waiting for late checkpoints. |
+| 130 | ~545 M | Full downstream eval + intrinsic metrics/diagnostics if held-out JSONL is staged. |
+| 260 | ~1.09 B | Full downstream eval + intrinsic metrics/diagnostics. |
+| 390 | ~1.64 B | Full downstream eval + intrinsic metrics/diagnostics; enters the selection window. |
+| 455 or final | ~1.91-2.00 B | Full downstream eval + intrinsic metrics/diagnostics; final selection evidence. |
+
+The final arm choice should use the late-window checkpoints (390 plus the final one, and 260 as the nearest pre-window anchor if only two late checkpoints are available). The iteration-65 Greek-only run is not selection evidence; it is an operational canary to catch broken conversion/eval early.
+
+### TD layer11 direct-path eval handoff
+
+The Token Distillation 2B run is a single selected arm, not the original
+`${RUN_TAG}_${arm}` three-arm layout. Use the TD-specific wrappers so conversion
+reads the direct checkpoint root and the HF tokenizer/config produced by the TD
+layer11 R17 roundtrip:
+
+```bash
+RUN_TAG=td_full25_layer11_2b_20260523T165038Z \
+ITER=65 \
+TASK_GROUP=greek_only \
+POLL_SECONDS=300 \
+nohup bash watch_and_submit_td_checkpoint_eval.sh \
+  > /capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter65_greek_only.log 2>&1 &
+```
+
+For selection checkpoints, enable intrinsic metrics against the staged Greek
+held-out slice:
+
+```bash
+RUN_TAG=td_full25_layer11_2b_20260523T165038Z \
+ITER=390 \
+TASK_GROUP=full \
+SUBMIT_INTRINSIC=1 \
+EVAL_JSONL=/iopsstor/scratch/cscs/fffoivos/cpt_corpus/heldout/cpt_greek_heldout_500_20260522.jsonl \
+POLL_SECONDS=300 \
+nohup bash watch_and_submit_td_checkpoint_eval.sh \
+  > /capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter390_full.log 2>&1 &
+```
+
+The TD wrapper submits `convert_bakeoff_checkpoint_to_hf.sbatch` with
+`ARM=td_layer11` and defaults the tokenizer/config to:
+
+`/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/hf_roundtrip`
+
+Checkpoint conversion is not a dataset-build/preprocessing job: it imports
+Megatron/PyTorch and requires the PyTorch `uenv`, which Clariden `xfer` nodes do
+not expose. It therefore runs as a bounded one-GPU `debug` job. The sleeping
+checkpoint watchers remain CPU-only `xfer` jobs.
+
+For unattended monitoring, prefer the Slurm wrapper instead of keeping
+sleeping watcher processes on a login SSH session:
+
+```bash
+first="$(sbatch --parsable watch_td_checkpoint_evals.sbatch)"
+second="$(sbatch --parsable --dependency=afterany:$first watch_td_checkpoint_evals.sbatch)"
+echo "$first $second"
+```
+
+The wrapper runs on `xfer` with one CPU and no GPU. It is safe to chain or
+re-run because each per-checkpoint watcher has a `submitted` stamp.
+
+## §5.6 hard gates — to be filled from V4 baseline
+
+Per v0.7 §5.6, a candidate arm **fails** if any of these gates trips. Thresholds are deliberately left as placeholders here — they're set **after** the V4 baseline run on unmodified Apertus-8B-2509 establishes the per-benchmark variance. The "fill from V4" step is on the post-V4 review checklist; until then these thresholds remain `PENDING(V4)`.
+
+| # | Hard gate | Signal source | Failure rule | Threshold |
+|---|---|---|---|---|
+| HG1 | English / core retention regression | `results.json` from lm-eval-harness (ARC, HellaSwag, WinoGrande, PIQA, MMLU) | mean drop > X p.p. on any one of those 5, after windowed average over last 3-5 checkpoints in 80-100 % of budget | `X = PENDING(V4)` (suggest 3 p.p. as starting placeholder) |
+| HG2 | Code retention regression (if code is a release requirement per Q A1) | lm-eval-harness code task (typically `humaneval` post-training; not in pretraining-eval table) | mean drop > Y p.p. | `Y = PENDING(V4 + Q A1)`; skip if code isn't a release requirement |
+| HG3 | New-token row collapse (cosine clustering, near-zero usage) | [`compute_new_token_diagnostics.py`](compute_new_token_diagnostics.py) → `embedding.new_E_cos.mean_off_diag` AND `forward.d2_avg_prob_mass_new_per_pos` | `mean_off_diag > C_THRESH` OR `prob_mass_new < M_THRESH` | `C_THRESH = PENDING(V4)` (suggest 0.5 as starting placeholder — base Greek tokens typically sit at ~0.1-0.3 off-diag mean); `M_THRESH = PENDING(V4)` |
+| HG4 | Polytonic text worsens vs base on character-normalized loss | [`compute_tokenizer_fair_metrics.py`](compute_tokenizer_fair_metrics.py) → `per_register.polytonic.nll_per_char` | arm value > base value | direct comparison; no separate threshold |
+| HG5 | Throughput / memory hit disproportionate to Greek compression gain | Megatron training logs (`--log-throughput`) + tokenizer-fair `compression_ratio` | `(throughput_loss / compression_gain) > T_THRESH` | `T_THRESH = PENDING(V4)`; only fires for extended-vocab arms (ReTok/Centroid), not Vanilla |
+| HG6 | Language-ID drift — model over-emits Greek in non-Greek prompts | Custom eval (§6.2; not yet implemented — Item 7 in COMPLETENESS_CHECK.md) | drift rate > D_THRESH | **`D_THRESH = PENDING(V4 + custom-eval-construction)`**; DEFERRED until §6.2 custom evals exist |
+
+**Operational note.** HG1-HG5 can be computed from JSONs already produced by the existing eval flow. HG6 requires the custom Greek eval construction work (1-2 weeks per v0.7); it's expected to not be running in time for the first bakeoff pass and may need a substitute signal (e.g., `per_register.english.nll_per_char` from `compute_tokenizer_fair_metrics.py` on an English held-out slice — non-Greek text getting Greek-token outputs would spike English NLL).
+
+**Why these thresholds aren't hard-coded:** the per-benchmark variance on Apertus-8B-base is unknown until we run V4. A "3 p.p. drop is a failure" rule is meaningless if the run-to-run variance of HellaSwag is itself 2 p.p. The V4 baseline + bootstrap CIs establish the noise floor, then "drop > 3× the noise floor on any benchmark" or similar becomes a defensible rule.
+
+**Selection (not automated).** For non-failing arms, v0.7 §5.6 gives a weighted score (30-40 % Greek BPC, 25-35 % Greek benchmarks, 10-15 % polytonic, 15-25 % retention, 5-10 % efficiency). For a 3-arm bakeoff this is small enough to eyeball — see [`summarize_bakeoff.py`](summarize_bakeoff.py) for the helper that aggregates per-arm JSONs into one markdown table. **The final pick is a manual review against this table + the V4 thresholds**, not a numerical aggregate.
+
+## Statistical methodology
+
+Per v0.7 §6.1 (Park et al. Oct 2025): downstream benchmark scores are noisy, and "run 3×" doesn't add information because the benchmarks are deterministic. The right way to get confidence intervals is **bootstrap over eval samples**.
+
+`compute_bootstrap_cis.py` (this dir) takes `--log_samples` JSONL from lm-eval-harness, resamples eval items with replacement 1000 times, and reports per-task mean + 95 % CI.
+
+**Selection scope** (per v0.7 §5.6): windowed average across the last 3–5 checkpoints in the 80–100 % range of each arm's budget. For training continuation post-winner, use the **raw** checkpoint at the end (not the averaged one), since the averaged model has no corresponding optimizer state.
+
+## Resources per eval run
+
+V4 baseline (one run) on Clariden:
+
+- partition: `normal`
+- shape: `-N 1 -t 4:00:00 --gpus-per-node=1` (V4 currently runs 1-GPU; the early 4-GPU plan was scaled back after PF5 / task-name corrections cut total work)
+- wall: ~1-2 h end-to-end for the seven Greek tasks + nine retention tasks at the corrected list (V4 job 2334245 mid-run as of 2026-05-21 12:30 UTC)
+- batch size: `auto` (lm-eval-harness picks per task; observed `auto:1` → `32` for Apertus-8B on bf16)
+
+### lm-eval-harness install path (Clariden, 2026-05-21)
+
+The uenv `pytorch/v2.9.1:v2`'s site-packages is read-only, so `pip install -e .` from the swissai clone fails with `[Errno 30] Read-only file system`. Workaround used by `run_eval.sbatch`:
+
+```bash
+# One-time, as part of CSCS staging:
+uenv start pytorch/v2.9.1:v2 --view=default
+cd /iopsstor/scratch/cscs/fffoivos/code/eval/lm-evaluation-harness-swissai
+pip install --target=/iopsstor/scratch/cscs/fffoivos/python_envs/lm_eval --quiet .
+pip install --target=/iopsstor/scratch/cscs/fffoivos/python_envs/lm_eval --no-deps --quiet accelerate
+# Delete the target's copy of huggingface-hub so the uenv's 0.36.0 wins over the
+# pip-installed 1.x (transformers 4.57.0 pins hf_hub<1.0):
+rm -rf /iopsstor/scratch/cscs/fffoivos/python_envs/lm_eval/huggingface_hub
+rm -rf /iopsstor/scratch/cscs/fffoivos/python_envs/lm_eval/huggingface_hub-*.dist-info
+```
+
+Then run_eval.sbatch sets `PYTHONPATH=/iopsstor/scratch/cscs/fffoivos/python_envs/lm_eval` and invokes `python3 -m lm_eval ...`. No `pip install -e .` at job-start time.
+
+Per-arm bakeoff eval at one checkpoint:
+
+- same shape, ~2 h
+- run inline during training via Slurm `--dependency=afterok` on the checkpoint-save step, OR run separately after the training job completes
+
+## Files in this directory
+
+- `EVAL_RECIPE.md` — this doc
+- `pull_benchmarks.sh` — staging-time HF download for the benchmark datasets (and the swiss-ai eval scripts repo)
+- `run_eval.sbatch` — parameterized eval job: `MODEL_PATH=… OUTPUT_DIR=… TASK_GROUP={full|greek_only|retention_only} sbatch run_eval.sbatch`
+- `run_apertus_baseline.sh` — thin wrapper: V4 baseline on the unmodified Apertus-8B-2509 checkpoint
+- `run_bakeoff_arm_eval.sh` — thin wrapper: per-arm eval, takes an arm's checkpoint dir as arg
+- `convert_bakeoff_checkpoint_to_hf.sbatch` — bounded one-GPU `debug` job; converts one Megatron `torch_dist` bakeoff checkpoint to HF format for eval
+- `run_megatron_convert_with_pg.py` — initializes the single-rank process group needed by Megatron `loader core` when reading `torch_dist` checkpoints
+- `submit_bakeoff_checkpoint_eval.sh` — submits conversion plus lm-eval, with optional intrinsic metrics when `SUBMIT_INTRINSIC=1`
+- `watch_and_submit_checkpoint_evals.sh` — lightweight watcher that stamps per-arm submissions and prevents duplicate checkpoint eval launches
+- `submit_td_checkpoint_eval.sh` — direct-path TD layer11 conversion plus eval submitter for the selected TD 2B run
+- `watch_and_submit_td_checkpoint_eval.sh` — lightweight direct-path TD watcher; submits once when a checkpoint directory and tracker are both ready
+- `watch_td_checkpoint_evals.sbatch` — CPU-only `xfer` wrapper that runs the TD watcher set durably under Slurm
+- `build_cpt_heldout_jsonl.py` / `build_cpt_heldout_jsonl.sbatch` — builds the 500-doc Greek held-out JSONL from the post-Apertus-dedup selected pool while excluding Greek doc_ids already used in `bulk_mix.jsonl`
+- `compute_bootstrap_cis.py` — post-process: bootstrap CIs over the `--log_samples` outputs
+- **`compute_tokenizer_fair_metrics.py`** — primary v0.7 §5.1 intrinsic metrics (BPC, NLL/char, NLL/word, tokens/word, chars/token, compression ratio, STRR). The cross-tokenizer-fair signal for comparing Vanilla (vocab 131,072) vs ReTok/Centroid (vocab 148,480). Has a `--stats-only` mode for tokenizer-only checks (no model load).
+- **`run_tokenizer_fair_metrics.sbatch`** — sbatch wrapper for the above; 1 node × 1 GPU × 2 h. Runs at each bakeoff checkpoint where downstream eval also runs.
+- **`compute_new_token_diagnostics.py`** — v0.7 §5.3 new-token integration diagnostic suite: 7 diagnostics over the 17,408 new IDs (rank of new target in next-token logits, prob-mass on new IDs, per-register entropy, top-1 substitution rate at new-target positions, greedy-gen new-token utilization, embedding L2-norm distribution new-vs-existing, cosine-similarity / effective-rank of new rows). Has `--embedding-only` mode that skips the forward-pass diagnostics (D1-D5) for cheap embedding-only health checks.
+- **`run_new_token_diagnostics.sbatch`** — sbatch wrapper for the diagnostic suite; 1 node × 1 GPU × 2 h.
+
+## §5.3 new-token integration diagnostic suite — important
+
+Per v0.7 §5.3, these 7 diagnostics are **"read at every bakeoff checkpoint"**:
+
+| # | Diagnostic | What it catches |
+|---|---|---|
+| D1 | Rank of correct new token in next-token logits | New token invisible |
+| D2 | Aggregate probability mass on new Greek tokens | Under- or over-emitted |
+| D3 | New-token entropy by register | Polytonic rows collapsed or avoided |
+| D4 | Top-k substitutions between new token and old subpieces | Model still prefers old segmentation |
+| D5 | Greedy-gen new-token utilization rate | New rows exist but behaviorally dead |
+| D6 | Embedding L2-norm distribution: new vs existing | Degenerate-subspace collapse |
+| D7 | Cosine-similarity / effective-rank of new rows | Same-direction collapse |
+
+`compute_new_token_diagnostics.py` implements all 7 in a single run (~10-15 min per checkpoint on one GH200). D6 + D7 are embedding-only (cheap; can run with `--embedding-only` for fast checks). D5 is the heaviest single diagnostic; `--skip-greedy` opts out.
+
+## Primary intrinsic metrics (v0.7 §5.1) — important
+
+Per-token PPL is **not comparable** across the Vanilla arm (vocab 131,072) and the ReTok/Centroid arms (vocab 148,480). v0.7 §5.1 specifies these tokenizer-fair metrics as the **primary** signal for the bakeoff:
+
+| Metric | Why |
+|---|---|
+| **BPC (bits per byte)** | Cleanest cross-tokenizer comparison |
+| **NLL per Unicode character** | More interpretable for Greek/polytonic |
+| **NLL per word** | Human-facing language metric |
+| **tokens/word, chars/token, compression ratio** | Quantifies tokenizer efficiency |
+| **STRR** (Subword-Tokenization Recovery Rate) | Fraction of held-out words that tokenize to a single token — whole-word preservation |
+
+`compute_tokenizer_fair_metrics.py` computes all of these from one HF-format checkpoint + a held-out JSONL. Aggregates globally + per-source + per-register.
+
+**Held-out eval slice**: ~100-500 docs covering modern Greek registers (encyclopedic / literary / academic / dialogue / legal / dictionary). The slice should be **outside** the bakeoff training mix. Current operational builder:
+
+```bash
+sbatch build_cpt_heldout_jsonl.sbatch
+```
+
+This samples from `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/cpt/selected_after_apertus_and_internal_dedup.parquet`, excludes every Greek `doc_id` already present in `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.jsonl`, and fills quotas from the Greek source/register buckets in `corpus_build/recipes/bulk.json` that still have training-disjoint rows. The first full scan showed the current training mix exhausted the literary and dictionary/misc filters, so the default 500-doc slice redistributes those slots to HPLT, dialogue/textbooks, academic, and legal/civic. This is more aligned with the current CPT corpus than the older C3 val/test reconstruction, but it is a **training-disjoint intrinsic-metrics slice**, not a proof that all external benchmark test items were absent from the CPT source pool. Benchmark decontamination remains a separate check.

provenance/evals/V4_BENCHMARK_COMPARISON.md

@@ -0,0 +1,77 @@
+# V4-HF vs V4-post-conversion — Apertus-8B-2509 benchmark comparison
+
+Companion to [`v4_baseline_corrected_20260521/README.md`](v4_baseline_corrected_20260521/README.md) and [`v4_postconv_retry_20260521/README.md`](v4_postconv_retry_20260521/README.md). Quantifies the R17 risk empirically: same Apertus weights, run through HF → Megatron → HF, then re-evaluated.
+
+## Per-task comparison
+
+Top-level group scores per swissai lm-evaluation-harness. Stderrs are the harness defaults (analytic, not bootstrapped — see `compute_bootstrap_cis.py` for per-sample CIs if needed).
+
+### English retention (Apertus Table 14)
+
+| Task | Metric | V4-HF | V4-postconv | Δ | Chance |
+|---|---|---:|---:|---:|---:|
+| `arc_challenge` | acc_norm | 0.5870 ±0.0144 | 0.2619 ±0.0128 | **−0.3251** | 0.25 |
+| `arc_easy` | acc_norm | 0.8363 ±0.0076 | 0.2614 ±0.0090 | **−0.5749** | 0.25 |
+| `hellaswag` | acc_norm | 0.7884 ±0.0041 | 0.2675 ±0.0044 | **−0.5209** | 0.25 |
+| `winogrande` | acc | 0.6930 ±0.0130 | 0.5107 ±0.0140 | −0.1823 | 0.50 |
+| `piqa` | acc_norm | 0.7992 ±0.0093 | 0.5212 ±0.0117 | −0.2780 | 0.50 |
+| `mmlu` | acc | 0.5923 ±0.0039 | 0.2295 ±0.0035 | **−0.3628** | 0.25 |
+
+`mmlu = 0.5923` on V4-HF matches the Apertus tech report Table 14 (≈0.59), confirming the eval pipeline is wired correctly.
+
+### Multilingual retention
+
+| Task | Metric | V4-HF | V4-postconv | Δ | Chance |
+|---|---|---:|---:|---:|---:|
+| `global_mmlu` (15 langs) | acc | 0.5246 ±0.0063 | 0.2381 ±0.0054 | **−0.2865** | 0.25 |
+| `xcopa` (11 langs) | acc | 0.6575 ±0.0063 | 0.5185 ±0.0067 | −0.1389 | 0.50 |
+| `xnli` (15 langs) | acc | 0.4400 ±0.0026 | 0.3321 ±0.0024 | −0.1079 | 0.33 |
+
+### Greek slice
+
+| Task | Metric | V4-HF | V4-postconv | Δ | Chance |
+|---|---|---:|---:|---:|---:|
+| `global_mmlu_full_el` | acc | 0.5155 ±0.0040 | 0.2295 ±0.0035 | **−0.2861** | 0.25 |
+| `include_base_44_greek_few_shot_en` | acc | 0.5054 ±0.0208 | 0.1975 ±0.0169 | **−0.3080** | 0.25 |
+| `belebele_ell_Grek` | acc | 0.6367 ±0.0160 | 0.2289 ±0.0140 | **−0.4078** | 0.25 |
+| `arc_challenge_mt_el` | acc_norm | 0.4795 ±0.0146 | 0.2637 ±0.0129 | **−0.2159** | 0.25 |
+| `xnli_el` | acc | 0.3984 ±0.0098 | 0.3333 ±0.0094 | −0.0651 | 0.33 |
+| `xquad_el` | f1 | 0.5172 ±0.0117 | **0.0000 ±0.0000** | **−0.5172** | n/a |
+| `global_piqa_completions_ell_grek` | acc_norm | 0.6200 ±0.0488 | 0.5400 ±0.0501 | −0.0800 | 0.50 |
+
+## Reading
+
+1. **V4-HF reproduces Apertus's published baseline.** Eval rig validated.
+2. **R17 is catastrophic, not cosmetic.** R1's "standard max abs diff = 0.0" was bit-exact only on the standard transformer tensors — the 128 reset xIELU αp/αn/β/ε parameters (per layer × 32 layers × 4 params) destroy the model's learned representations. Closed-form MC tasks fall to or below chance; extractive QA (`xquad_el`) collapses to zero f1 because the model cannot produce coherent text spans at all.
+3. **xIELU is the load-bearing R17 component.** xIELU's α-parameters are multiplicative slope gates: resetting them to the `XIELU.__init__` default (αp = αn = 0.8) destroys the learned activation shape across all 32 layers. The QK-Norm `q_norm`/`k_norm` weights happened to drift inside bf16 noise in the R1 diff (within 1e-3 threshold), but the xIELU resets are larger and they propagate through every forward pass.
+4. **Tasks that "survive" R17 are easy-baseline tasks.** `winogrande`, `piqa`, `xcopa`, `xnli`, `global_piqa_completions_ell_grek` retain near-chance behaviour because the random init still produces *some* output and binary/3-way MC is forgiving. The information-rich tasks (MMLU, ARC, HellaSwag, Belebele, XQuAD) collapse.
+
+## Bakeoff selection implications
+
+This section describes the implication **for raw, unpatched HF → Megatron conversion**. After this comparison exposed the failure, `megatron_patches/patch_apertus_extras.py` was implemented and validated on the TP=2 bakeoff init checkpoints (`2341182`, `2341239`, `2341241`): patched Megatron → HF roundtrips have `standard_max_abs_diff = 0.0`, `r17_max_abs_diff = 0.0`, and `logit_max_abs_diff = 0.0`.
+
+Therefore the actual bakeoff should use `megatron_tp2_r17patched`, not the raw `megatron_tp2` dirs. If someone intentionally runs the raw dirs, then the three arms start from approximately the V4-postconv baseline (R17-reset Apertus + the embedding-resize init choice they differ on), not from V4-HF. In that raw-conversion ablation, the §5.6 hard-gate plan in [`EVAL_RECIPE.md`](EVAL_RECIPE.md) would need to change:
+
+- **Old framing (V4-HF reference):** "Arm X regresses > 3 p.p. from V4-HF on mmlu" → trivially fails for every arm at iteration 0 (each is 36 p.p. below V4-HF by construction).
+- **New framing (V4-postconv reference):** "Arm X recovers > Y p.p. from V4-postconv on mmlu after 2B tokens, with bootstrap CI separating it from the other arms" → measurable signal.
+
+The bakeoff is **recovery**, not retention. Concrete questions per arm:
+
+- Which arm climbs off the chance floor fastest on the closed-form MC tasks (mmlu, hellaswag, arc_easy)?
+- Which arm preserves the partially-surviving signals (winogrande, piqa, xcopa) without backsliding?
+- Which arm recovers `xquad_el` extractive QA — i.e. produces non-empty answers?
+- Greek-specific recovery is the primary bakeoff signal: which arm recovers `belebele_ell_Grek` and `global_mmlu_full_el` fastest?
+
+If after 2B tokens no arm has recovered meaningfully past V4-postconv (e.g. all three sit at chance ±0.01 on mmlu), the bakeoff doesn't differentiate at this budget; either extend the budget or block on the production patcher (`megatron_patches/patch_apertus_extras.py`) that fixes R17 before training.
+
+## Production CPT prerequisite
+
+R17 quantification confirmed the existing risk note in [`RISKS.md`](../../../RISKS.md): **the post-conversion xIELU/QK-Norm patcher is required before production CPT**, not just nice-to-have. Recovering 30-50 percentage points across MMLU/HellaSwag/ARC inside a 15-20B-token production CPT budget is unrealistic; the model must start from intact xIELU values.
+
+The patcher is now implemented and has passed tensor + logit roundtrip checks for all three TP=2 init arms. The remaining optional confirmation is to run the full V4 eval on a patched roundtrip; the cheaper proof already shows the patched HF roundtrip is bit-exact to its HF source.
+
+## Artifacts
+
+- `v4_baseline_corrected_20260521/results.json` — V4-HF full lm-eval-harness output.
+- `v4_postconv_retry_20260521/{retention,greek}_results.json` — V4-postconv split runs (retention on job 2338020, Greek on job 2338021).
+- Comparison script: `/tmp/v4_compare_full.py` (paths in script point at this dir).

provenance/token-distillation/RUN_LOG_20260523.md

@@ -0,0 +1,2299 @@
+# Token Distillation Coverage Run Log - 2026-05-23
+
+Purpose: run the first bounded Token Distillation gate, the CPU-only firing and
+snippet-coverage prepass, on Clariden `xfer`. No GPU nodes are used for this
+dataset/snippet work.
+
+## Local commits
+
+- `b6d412a` - production decision state and initial TD prepass.
+- `462ae2f` - first attempt to fix `uenv` path in Slurm.
+- `b1aec29` - switch TD prepass launcher to a Python venv instead of `uenv`.
+- `a05e3ab` - document xfer runtime and xfer-built venv.
+- `77119b3` - add periodic progress logging.
+- `65d8561` - document full TD coverage prepass launch.
+- `c17c8ed` - pin vendored upstream Token Distillation source.
+- `fcddd50` - add coverage-to-smoke/layer-pilot token selector.
+- `dd9a27b` - add dependent TD coverage postprocess job.
+- `6b2473c` - add TD preservation verifier.
+- `bec2d02` - make TD preservation verifier xfer-safe.
+- `993a82d` - support xfer Python in TD verifier.
+- `bfa8a3c` - log full TD monitoring gates.
+- `b23de6a` - update TD production gate state.
+- `276afed` - log layer11 full TD checkpoint.
+- `9c34f2f` - log full TD preservation pass.
+- `957c46b` - add full TD preservation artifacts.
+- `d6e7f20` - add full TD intrinsic eval results.
+- `83d6934` - record full TD eval decision.
+- `86da8f0` - add TD layer11 R17 roundtrip gate.
+- `0143e01` - record TD layer11 R17 roundtrip pass.
+
+## TD0 pinned implementation source
+
+Pinned upstream Token Distillation source for the next adapter step:
+
+- upstream: `https://github.com/konstantinjdobler/token-distillation`
+- commit: `35702b5809599ecd68b7845eca27a0d7b7cec0da`
+- local vendored subset:
+  `external/token-distillation/`
+
+Important integration note: do not call the package's high-level
+`TokenDistillation.run(...)` entry point for Apertus production. It appends
+tokens with `add_tokens(...)`; our ReTok arm uses an exact merge-extended
+tokenizer with fixed IDs. The Apertus adapter should call the lower-level
+training loop with an explicit `[131072, 148480)` mapping.
+
+## Failed handoffs and fixes
+
+| Job | State | Cause | Fix |
+|---:|---|---|---|
+| `2349182` | failed | `uenv: command not found` inside xfer batch node | stopped using implicit `uenv` CLI |
+| `2349183` | failed | `/usr/bin/uenv` exists on login but not on xfer compute node | confirmed xfer cannot rely on `uenv` |
+| `2349184` | completed diagnostic | printed xfer PATH; confirmed no `uenv` on node `nid001306` | moved away from uenv |
+| `2349185` | failed diagnostic | xfer system Python is 3.6 and lacks `tokenizers` | tested Python 3.11 path |
+| `2349211` | failed smoke | venv was built on login (`aarch64`) but xfer is `x86_64`, so `tokenizers.abi3.so` could not load | created xfer-built venv |
+| `2349237` | completed setup | created `/iopsstor/scratch/cscs/fffoivos/python_envs/td_coverage_py311_xfer` on xfer; installed `tokenizers==0.22.1` | launcher now defaults to this venv |
+| `2349241` | completed smoke | 50k-token smoke produced coverage JSONL, summary JSON, and snippet index | proved launcher and artifact path |
+| `2349259` | completed smoke | 50k-token smoke with `PROGRESS_TOKEN_INTERVAL=10000` printed progress JSON lines | proved long-run health logging |
+| `2349342` | failed after full scan | strict prepass found 9 non-NFC docs in the first 2B-token scan and refused to emit coverage | added CPU-only JSONL NFC normalizer and reran prepass on `bulk_mix.nfc.jsonl` |
+
+## First full run attempt
+
+Job: `2349342`
+
+Command shape:
+
+```bash
+cd /iopsstor/scratch/cscs/fffoivos/repo/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/token_distillation
+sbatch --export=ALL,\
+OUTPUT_DIR=/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T012000Z_r2,\
+INPUT_JSONL=/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.jsonl,\
+TARGET_EXTENDED_TOKENS=2000000000,\
+SNIPPETS_PER_TOKEN=100,\
+PROGRESS_TOKEN_INTERVAL=5000000 \
+td_coverage_prepass_xfer.sbatch
+```
+
+Stable output directory:
+
+`/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T012000Z_r2`
+
+Expected artifacts:
+
+- `td_coverage_prepass.jsonl`
+- `td_coverage_summary.json`
+- `td_snippet_index/snippets.jsonl`
+
+Health at first check:
+
+- Slurm state: `RUNNING` on `xfer`, node `nid001154`.
+- Elapsed: `00:01:35`.
+- `sstat` RSS: `2581400K`.
+- Progress lines:
+  - `tokens_scanned=5042268`, `docs_seen=3244`
+  - `tokens_scanned=10003445`, `docs_seen=6118`
+
+Approximate early throughput from progress lines: about 100k extended tokens/s,
+so the 2B scan should be on the order of 5-6 hours if throughput remains
+stable. This is an estimate, not a completion promise.
+
+Health at 2026-05-22T23:29Z:
+
+- Slurm state: `RUNNING` on `xfer`, node `nid001154`.
+- Elapsed: `00:26:53`.
+- Latest progress: `tokens_scanned=200000773`, `docs_seen=146900`.
+- Stderr: empty.
+- No final artifacts yet; `td_coverage_summary.json` is written only at
+  completion.
+
+Health at 2026-05-22T23:37Z:
+
+- Slurm state: `RUNNING` on `xfer`, node `nid001154`.
+- Elapsed: `00:35:07`.
+- Latest progress: `tokens_scanned=265000226`, `docs_seen=201079`.
+- Stderr: empty.
+
+Health at 2026-05-22T23:50Z:
+
+- Slurm state: `RUNNING` on `xfer`, node `nid001154`.
+- Elapsed: `00:48:28`.
+- Latest progress: `tokens_scanned=365000047`, `docs_seen=285789`.
+- Stderr: empty.
+- No final artifacts yet.
+- Dependent CPU-only postprocess queued as job `2349550`
+  (`afterok:2349342`). It will render `TD_COVERAGE_SUMMARY.md`, write
+  `TD_GATE_DECISION.json`, and create `pilot_selection/` only if the coverage
+  gate recommends TD.
+
+Final state:
+
+- Slurm state: `FAILED`, elapsed `04:14:40`, exit `1:0`.
+- Last progress before failure:
+  `tokens_scanned=1995095832`, `docs_seen=1641638`,
+  `chars_seen=7468063507`.
+- Error:
+  `Refusing to emit TD coverage: 9 docs were not NFC. Run on the post-normalize corpus or pass --no-require-nfc for a diagnostic-only scan.`
+- The dependent postprocess job `2349550` was left with
+  `DependencyNeverSatisfied` and was cancelled.
+
+Interpretation: this is a corpus invariant failure, not a TD coverage result.
+Do not use `--no-require-nfc` for the decision gate.
+
+## NFC JSONL repair and rerun chain
+
+Added:
+
+- `corpus_build/normalize_jsonl_nfc.py`
+- `corpus_build/normalize_jsonl_nfc.sbatch`
+
+Local and Clariden validation:
+
+- `python3 -m py_compile corpus_build/normalize_jsonl_nfc.py`
+- `bash check_cpu_only_slurm.sh` passes with the JSONL normalizer included.
+- Tiny local smoke confirmed only non-NFC text rows are rewritten.
+
+Normalization job:
+
+- Job: `2351373`
+- Partition: `xfer`
+- Input: `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.jsonl`
+- Output: `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.jsonl`
+- Manifest: `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.manifest.json`
+- State: `COMPLETED`, elapsed `00:08:21`.
+
+Manifest highlights:
+
+- rows: `5,754,172`
+- non-NFC docs before normalization: `37`
+- changed docs: `37`
+- missing/non-string text rows: `0`
+- input bytes: `43,832,960,668`
+- output bytes: `43,832,959,909`
+- output SHA-256:
+  `205cb5ed80a6150064b290aaa0457463daf6c0e514e6c27b9ea3da28a479a8af`
+- sampled changed rows are from `code_codeparrot_clean`.
+
+Strict rerun chain:
+
+| Job | Purpose | State at launch |
+|---:|---|---|
+| `2351374` | TD coverage prepass on `bulk_mix.nfc.jsonl` | running on `xfer` |
+| `2351375` | TD coverage postprocess / gate | pending `afterok:2351374` |
+
+Rerun output directory:
+
+`/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc`
+
+## TD smoke wrapper readiness
+
+Added `train_retok_td.py` and `train_retok_td.sbatch` for the second gate. They
+are not launched automatically; only use them if the full coverage prepass
+recommends TD.
+
+Clariden dry-run validation on the completed 50k smoke artifacts:
+
+- Selection artifact:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_wrapper_dry_run_20260523_low/selection/td_pilot_token_selection.json`
+- Dry-run manifest:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_wrapper_dry_run_20260523_low/dry_run/retok_td_manifest.json`
+- Result: loaded base and student tokenizers, grouped one real prepass snippet,
+  wrote manifest, and exited before model load/training.
+- Note: this used `--min-status low_lt20` only to exercise the wrapper against
+  the tiny 50k diagnostic prepass. The real TD gate remains `enough_25` or
+  better on the full 2B prepass.
+
+## Decision gate after completion
+
+Read `td_coverage_summary.json`:
+
+- `recommended_next_step = run_full_td_100`: proceed to TD smoke/layer pilot.
+- `recommended_next_step = run_td_25_with_flagged_tail`: run the paper-fast TD
+  setting and keep tail-token strategy explicit.
+- `recommended_next_step = do_not_launch_full_td_inspect_coverage`: do not spend
+  GPU time; inspect tokenizer/corpus coverage first.
+
+## Strict full-2B coverage result
+
+The strict rerun completed on the normalized JSONL:
+
+- Prepass job: `2351374`, partition `xfer`, state `COMPLETED`,
+  elapsed `04:09:23`.
+- Postprocess job: `2351375`, partition `xfer`, state `COMPLETED`,
+  elapsed `00:00:04`.
+- Coverage directory:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc`
+- Gate file:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc/TD_GATE_DECISION.json`
+
+Gate decision:
+
+```json
+{
+  "recommended_next_step": "run_full_td_100",
+  "selection_status": "created",
+  "enough_100_fraction": 0.9982192095588235,
+  "enough_25_fraction": 0.9990808823529411,
+  "low_lt25_count": 16,
+  "tokens_scanned": 2000000000,
+  "target_extended_tokens": 2000000000
+}
+```
+
+Summary:
+
+- Input corpus: `bulk_mix.nfc.jsonl`.
+- Extended tokens scanned: exactly `2,000,000,000`.
+- Docs used: `1,645,852`.
+- `require_nfc`: true.
+- `non_nfc_docs`: 0.
+- Status counts:
+  - `enough_100`: 17,377 tokens (99.82%).
+  - `enough_25`: 15 tokens (0.09%).
+  - `low_20_24`: 2 tokens.
+  - `low_lt20`: 3 tokens.
+  - `zero`: 11 tokens.
+- Action counts:
+  - `td_100`: 17,377 tokens.
+  - `td_25`: 15 tokens.
+  - `keep_retok`: 5 tokens.
+  - `inspect`: 11 tokens.
+
+Pilot selections were created under `pilot_selection/`:
+
+- `smoke_token_ids.txt`: 512 tokens.
+- `layer_pilot_token_ids.txt`: 2,048 tokens.
+
+## First TD smoke attempt
+
+Launched a bounded one-GPU last-layer smoke, not full TD:
+
+- Job: `2353063`
+- Partition: `normal`
+- Output directory:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_smoke_last_layer_20260523T074639Z`
+- Settings: `TARGET_LAYER=-1`, `SNIPPETS_PER_TOKEN=25`, `EPOCHS=1`,
+  `BATCH_SIZE=16`.
+
+Result:
+
+- State: `FAILED`
+- Elapsed: `00:00:29`
+- Failure:
+  `token 140716: no snippets contain its base phrase after base tokenization`
+
+Inspection showed token `140716` is an unstable byte-level/noisy token:
+
+- `raw_token`: `"âľ"`
+- `token_string`: `"�"`
+- `base_subtoken_ids`: `[66122]`
+- `extended_firings`: `132607`
+- `status`: `enough_100`
+
+The selected snippets' actual surfaces were assorted symbols, with different
+base-token spans. This is not a TD training result; it is a wrapper/selection
+invariant failure for unstable token mappings. These tokens should be logged,
+skipped, and preserved at their ReTok initialization rather than crashing a
+bounded smoke.
+
+## TD wrapper hardening
+
+Updated `train_retok_td.py` and `train_retok_td.sbatch` so a selected token is
+trained only if enough snippets reproduce its static base phrase under the base
+tokenizer. Tokens that fail this check are recorded in
+`retok_td_manifest.json` under `skipped_tokens` and remain frozen/preserved at
+their existing ReTok initialization.
+
+Default smoke behavior:
+
+- `MIN_ACCEPTED_SNIPPETS_PER_TOKEN=$SNIPPETS_PER_TOKEN`
+- `MIN_TRAINED_TOKEN_FRACTION=0.90`
+
+This means a smoke can tolerate a small number of unstable/noisy selected
+tokens, but fails before model load if too much of the selection is untrainable.
+
+Validation:
+
+- Local: `python3 -m py_compile train_retok_td.py`
+- Clariden with job runtime env:
+  `uenv run pytorch/v2.9.1:v2 --view=default -- python3 -m py_compile train_retok_td.py`
+
+Relaunched bounded smoke:
+
+- Job: `2353091`
+- Output directory:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_smoke_last_layer_20260523T075520Z`
+- Initial state: pending on `normal`.
+
+Result:
+
+- State: `FAILED`
+- Elapsed: `00:06:33`
+- Allocation note: requested `gres/gpu=1`, but Clariden allocated the full
+  node (`gres/gpu=4`). Do not launch larger single-process TD jobs in this
+  shape without packing work across the node or accepting the inefficiency.
+- The wrapper and model path succeeded:
+  - 510/512 selected tokens were trainable.
+  - 2 unstable byte-level/noisy tokens were skipped and preserved.
+  - Apertus/ReTok loaded and entered real training.
+- Failure mode: CUDA OOM at step 471/796 during backward.
+- OOM details:
+  - Tried to allocate `2.47 GiB`.
+  - GPU capacity `95.00 GiB`, free `1.83 GiB`.
+  - PyTorch allocated `76.45 GiB`, reserved-unallocated `15.94 GiB`.
+  - Error recommended `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`.
+
+Interpretation: this is not a TD token-selection failure. It is the memory cost
+of running upstream TD on Apertus-8B with `output_hidden_states=True`,
+MSE-on-hiddens, and the untied-output CE path at `BATCH_SIZE=16`.
+
+Follow-up fix:
+
+- Default `BATCH_SIZE` for the sbatch reduced from `16` to `8`.
+- `PYTORCH_ALLOC_CONF` defaults to `expandable_segments:True`.
+- Next rerun should stay bounded and should not expand beyond the smoke until
+  the scheduler packing issue is handled.
+
+## Batch-8 smoke completion
+
+Relaunched the same bounded 512-token last-layer smoke with batch 8:
+
+- Job: `2353128`
+- Output directory:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_smoke_last_layer_20260523T080555Z_bs8`
+- State: `COMPLETED`
+- Elapsed: `00:11:09`
+- Allocation: full node despite one-GPU request
+  (`AllocTRES=cpu=288,gres/gpu=4,mem=450G,node=1`).
+- Runtime settings:
+  - `TARGET_LAYER=-1`
+  - `SNIPPETS_PER_TOKEN=25`
+  - `BATCH_SIZE=8`
+  - `MIN_TRAINED_TOKEN_FRACTION=0.90`
+  - allocator config: `expandable_segments:True`
+
+Manifest:
+
+```json
+{
+  "requested_token_count": 512,
+  "trained_token_count": 510,
+  "skipped_token_count": 2,
+  "trained_token_fraction": 0.99609375,
+  "batch_size": 8,
+  "completed": true,
+  "elapsed_seconds": 657.6161992549896
+}
+```
+
+Output files include:
+
+- `model-00001-of-00004.safetensors`
+- `model-00002-of-00004.safetensors`
+- `model-00003-of-00004.safetensors`
+- `model-00004-of-00004.safetensors`
+- `model.safetensors.index.json`
+- tokenizer/config files
+- `retok_td_manifest.json`
+
+Smoke conclusion:
+
+- The ReTok + TD wrapper can train and save an Apertus-8B HF checkpoint using
+  the vendored TD library path.
+- For Apertus-8B, `BATCH_SIZE=16` is too close to memory limits; batch 8 plus
+  expandable segments works for the 512-token smoke.
+- Scheduler packing remains unresolved: larger TD experiments should not be
+  launched as single-process jobs that burn a full four-GPU node unless we
+  deliberately pack independent TD work into the same allocation or decide that
+  the cost is acceptable.
+
+## Layer-pilot preparation
+
+Checked the vendored package for a layer-suggestion tool. There is no separate
+tool in the pinned implementation; the actionable upstream guidance is the
+README note that target layers around the one-third mark can perform better
+than the last layer.
+
+Layer-pilot decision:
+
+- Candidate A: `target_layer=-1` (paper-default last layer).
+- Candidate B: `target_layer=11` (one-third-depth suggestion for Apertus-8B's
+  32 transformer layers).
+- Do not add a Candidate C unless a real Apertus-specific probe produces a
+  well-defined `L*`; do not invent one just to satisfy the plan wording.
+
+Added `train_retok_td_layer_pilot_packed.sbatch` to run these candidates inside
+one full-node allocation on separate GPUs. This addresses the observed
+Clariden behavior where one-GPU requests are allocated/billed as full 4-GPU
+nodes.
+
+## Packed layer pilot completion
+
+Launched the packed layer pilot as a bounded two-candidate run:
+
+- Job: `2353172`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_layer_pilot_20260523T082252Z`
+- Candidates:
+  - `target_layer=-1`, label `last`, GPU 0.
+  - `target_layer=11`, label `layer11`, GPU 1.
+- Selection: `1,024` requested tokens, `50` snippets per token,
+  `BATCH_SIZE=8`.
+- State: `COMPLETED`
+- Elapsed: `00:42:25`
+- Allocation: full node
+  (`AllocTRES=billing=288,cpu=288,energy=3916310,gres/gpu=4,mem=450G,node=1`).
+
+Top-level manifest:
+
+`/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_layer_pilot_20260523T082252Z/layer_pilot_manifest.json`
+
+Manifest highlights:
+
+| Target layer | Label | Requested | Trained | Skipped | Trained fraction | Elapsed |
+|---:|---|---:|---:|---:|---:|---:|
+| `-1` | `last` | 1,024 | 1,020 | 4 | 0.9961 | 2,533.7s |
+| `11` | `layer11` | 1,024 | 1,020 | 4 | 0.9961 | 1,966.9s |
+
+Both candidates saved HF checkpoints with tokenizer/config files and
+`model.safetensors.index.json`.
+
+Training-log note: `last` had low, stable reported loss around `0.017-0.022`.
+`layer11` completed but showed large intermittent loss spikes and a high running
+loss. This is a warning sign, not the selection criterion. The layer choice must
+come from heldout Greek BPC/NLL plus D1/D2/D4/D5 new-token diagnostics against
+the same ReTok baseline.
+
+## Packed intrinsic eval preparation
+
+Added:
+
+- `eval/run_td_pilot_intrinsics_packed.sbatch`
+- `eval/summarize_td_pilot_intrinsics.py`
+
+Purpose: evaluate the three relevant HF checkpoints in one full-node allocation:
+
+- ReTok baseline:
+  `/iopsstor/scratch/cscs/fffoivos/init_checkpoints/modern_only_148480/retok`
+- TD last-layer pilot:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_layer_pilot_20260523T082252Z/last`
+- TD layer-11 pilot:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_layer_pilot_20260523T082252Z/layer11`
+
+Heldout eval file:
+
+`/iopsstor/scratch/cscs/fffoivos/cpt_corpus/heldout/cpt_greek_heldout_500_20260522.jsonl`
+
+The packed eval runs:
+
+- `compute_tokenizer_fair_metrics.py` for BPC/NLL/compression/STRR.
+- `compute_new_token_diagnostics.py` for D1/D2/D4/D5 plus embedding-space
+  D6/D7 checks.
+
+This keeps the TD layer decision bounded and avoids three separate full-node
+eval allocations.
+
+## Packed intrinsic eval result
+
+Submitted packed eval:
+
+- Job: `2353771`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_pilot_intrinsics_20260523T091637Z`
+- State: `COMPLETED`
+- Elapsed: `00:03:09`
+- Allocation: full node
+  (`AllocTRES=billing=288,cpu=288,energy=296006,gres/gpu=4,mem=450G,node=1`).
+
+Local copies of the compact result:
+
+- `eval/td_pilot_intrinsics_20260523T091637Z/TD_PILOT_INTRINSICS_SUMMARY.md`
+- `eval/td_pilot_intrinsics_20260523T091637Z/td_pilot_intrinsics_summary.json`
+- `eval/td_pilot_intrinsics_20260523T091637Z/model_spec.tsv`
+
+Primary metrics:
+
+| Arm | BPC | Delta vs ReTok | D1 mean rank | D1 top1 | D1 top5 | D2 new mass | D4 top1-new | D5 greedy-new |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| ReTok | 2.9503 | 0.0000 | 3868.27 | 0.0065 | 0.0231 | 0.9821 | 0.9992 | 1.0000 |
+| TD last | 2.7830 | -0.1673 | 3294.05 | 0.0146 | 0.0383 | 0.9796 | 0.9991 | 1.0000 |
+| TD layer 11 | 2.7753 | -0.1750 | 3263.69 | 0.0148 | 0.0385 | 0.9795 | 0.9991 | 1.0000 |
+
+Interpretation:
+
+- Both TD pilot checkpoints materially improve heldout BPC and D1 rank over the
+  ReTok baseline.
+- `target_layer=11` is slightly better than `target_layer=-1` on BPC and D1.
+- The earlier `layer11` training-loss spikes did not show up as output-norm,
+  D4, or D5 instability in this intrinsic eval.
+- Selected next TD target layer for the full-token challenger: `target_layer=11`.
+
+Resource calibration:
+
+- `layer11` pilot runtime was about `1,966.9s` for `1,020` trained tokens at
+  `50` snippets per token.
+- A naive all-token `100`-snippet run would be roughly `18-19` wall hours on
+  one GPU, beyond `normal`'s 12h limit and wasteful on Clariden's 4-GPU node
+  allocations.
+- The next full-token challenger should therefore start with the paper-fast
+  `25`-snippet setting, not the `100`-snippet setting. A later `100`-snippet or
+  `50`-snippet run is only justified if the fast full-token TD checkpoint clears
+  the gates but leaves quality ambiguous.
+
+Selection artifact update:
+
+- Regenerated `pilot_selection/` with an added all-eligible ID list:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc/pilot_selection/full_td_token_ids.txt`
+- Full-token eligible count: `17,392`.
+
+## Full-token 25-snippet TD launch
+
+Launched the first full-token TD challenger using the paper-fast snippet budget:
+
+- Job: `2353960`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_full25_layers_20260523T092602Z`
+- Walltime: `10:00:00`
+- Token IDs:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc/pilot_selection/full_td_token_ids.txt`
+- Coverage JSONL:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc/td_coverage_prepass.jsonl`
+- Snippets JSONL:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/coverage_2b_modern_20260523T032424Z_nfc/td_snippet_index/snippets.jsonl`
+- Candidates packed into the same allocation:
+  - `target_layer=-1`, label `last`, GPU 0.
+  - `target_layer=11`, label `layer11`, GPU 1.
+- Runtime settings:
+  - `SNIPPETS_PER_TOKEN=25`
+  - `MIN_ACCEPTED_SNIPPETS_PER_TOKEN=25`
+  - `MAX_SELECTED_TOKENS=0` (full token list)
+  - `BATCH_SIZE=8`
+  - `MIN_TRAINED_TOKEN_FRACTION=0.99`
+
+Startup health:
+
+- Job state: `RUNNING` on `nid006119`.
+- GPUs visible: `0,1,2,3`; training processes started on GPUs 0 and 1.
+- Early process check showed both Python workers alive with GPU contexts
+  allocated (`~6.4 GiB` each) and high CPU during snippet preparation.
+
+## Full-token run babysitting and queued gates
+
+Live checks on 2026-05-23 confirmed the full-token TD job has not restarted:
+
+- Slurm job: `2353960`
+- `scontrol` field: `Restarts=0`
+- Node: `nid006119`
+- Active processes:
+  - `target_layer=-1` on GPU 0
+  - `target_layer=11` on GPU 1
+- GPU health check showed GPUs 0 and 1 saturated, while GPUs 2 and 3 were idle.
+  This is expected for the current two-candidate packed run; no extra CPU-only
+  work was placed on GPU nodes.
+
+Progress snapshots:
+
+| UTC time | last progress | last ETA | layer11 progress | layer11 ETA |
+|---|---:|---:|---:|---:|
+| 12:22 | 27,528 / 54,303 (50.7%) | 2:44:02 | 35,172 / 54,303 (64.8%) | 1:34:03 |
+| 12:27 | 28,322 / 54,303 (52.2%) | 2:36:34 | 36,186 / 54,303 (66.6%) | 1:29:37 |
+| 12:32 | 29,114 / 54,303 (53.6%) | 2:41:20 | 37,204 / 54,303 (68.5%) | 1:21:46 |
+| 12:37 | 29,907 / 54,303 (55.1%) | 2:36:01 | 38,228 / 54,303 (70.4%) | 1:17:50 |
+| 12:42 | 30,704 / 54,303 (56.5%) | 2:29:37 | 39,244 / 54,303 (72.3%) | 1:18:35 |
+| 12:47 | 31,497 / 54,303 (58.0%) | 2:22:13 | 40,267 / 54,303 (74.2%) | 1:08:12 |
+| 12:53 | 32,288 / 54,303 (59.5%) | 2:16:11 | 41,286 / 54,303 (76.0%) | 1:03:58 |
+| 12:58 | 33,081 / 54,303 (60.9%) | 2:13:56 | 42,304 / 54,303 (77.9%) | 1:00:16 |
+| 13:03 | 33,877 / 54,303 (62.4%) | 2:10:12 | 43,322 / 54,303 (79.8%) | 0:52:36 |
+| 13:08 | 34,672 / 54,303 (63.8%) | 2:01:25 | 44,338 / 54,303 (81.6%) | 0:47:20 |
+| 13:13 | 35,464 / 54,303 (65.3%) | 1:58:48 | 45,360 / 54,303 (83.5%) | 0:44:02 |
+| 13:18 | 36,258 / 54,303 (66.8%) | 1:51:51 | 46,378 / 54,303 (85.4%) | 0:37:28 |
+| 13:23 | 37,050 / 54,303 (68.2%) | 1:52:41 | 47,397 / 54,303 (87.3%) | 0:33:31 |
+| 13:28 | 37,844 / 54,303 (69.7%) | 1:41:42 | 48,415 / 54,303 (89.2%) | 0:28:47 |
+| 13:33 | 38,643 / 54,303 (71.2%) | 1:38:45 | 49,425 / 54,303 (91.0%) | 0:25:02 |
+| 13:38 | 39,440 / 54,303 (72.6%) | 1:32:13 | 50,446 / 54,303 (92.9%) | 0:19:23 |
+| 13:43 | 40,236 / 54,303 (74.1%) | 1:25:29 | 51,465 / 54,303 (94.8%) | 0:14:26 |
+| 13:48 | 41,033 / 54,303 (75.6%) | 1:25:07 | 52,485 / 54,303 (96.7%) | 0:08:24 |
+| 13:53 | 41,826 / 54,303 (77.0%) | 1:18:51 | 53,501 / 54,303 (98.5%) | 0:03:49 |
+| 13:58 | 42,623 / 54,303 (78.5%) | 1:12:21 | 54,303 / 54,303 (100.0%) | 0:00:00 |
+
+No `model.safetensors*` checkpoint files existed yet at these snapshots. The
+manifests had already been written by the wrapper:
+
+- `last/retok_td_manifest.json`
+- `layer11/retok_td_manifest.json`
+
+Both manifests report:
+
+- `trained_token_count = 17,377`
+- `skipped_token_count = 15`
+- `snippets_per_token = 25`
+- `min_accepted_snippets_per_token = 25`
+
+## Preservation verifier hardening
+
+Added and then hardened the post-TD preservation verifier:
+
+- Commit `6b2473c` - add `verify_td_preservation.py` and
+  `verify_td_preservation_xfer.sbatch`.
+- Commit `bec2d02` - remove the torch dependency by parsing Safetensors
+  directly, so the check can run on `xfer` as CPU-only work.
+- Commit `993a82d` - make the verifier compatible with `xfer`'s Python version
+  by replacing `int | None` annotations with `Optional[int]`.
+
+The verifier now checks, from checkpoint artifacts only:
+
+- state-dict key equality;
+- tensor shape and dtype equality;
+- exact byte preservation for every non-embedding tensor;
+- exact byte preservation for all input/output embedding rows except the
+  trained new-token IDs;
+- every trained new input/output row changed;
+- trained rows are not all-zero and have no BF16/F16/F32 inf/nan bit patterns;
+- xIELU and QK-Norm tensors are covered by the non-embedding exact check.
+
+Diagnostics run while hardening:
+
+| Job | Partition | Result | Meaning |
+|---:|---|---|---|
+| `2355602` | `xfer` | failed with `uenv: command not found` | confirmed `xfer` cannot use the PyTorch uenv; preservation must not depend on torch there |
+| `2355617` | `xfer` | exposed Python syntax incompatibility | first xfer compile/help check caught `int | None` on the older xfer Python |
+| `2355625` | `xfer` | completed | xfer compile/help check passed after the compatibility patch |
+
+Local tests:
+
+- `python3 -m py_compile verify_td_preservation.py`
+- `bash -n verify_td_preservation_xfer.sbatch`
+- synthetic tiny Safetensors roundtrip test of preserved rows, moved trained
+  rows, and unchanged xIELU-like tensor.
+
+The xfer-safe verifier was synced to:
+
+`/iopsstor/scratch/cscs/fffoivos/repo/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/token_distillation/`
+
+## Queued post-TD gates
+
+Queued xfer preservation jobs behind the full TD job:
+
+| Arm | Job | Partition | Dependency | Output |
+|---|---:|---|---|---|
+| `last` | `2355706` | `xfer` | `afterok:2353960` | `last/td_preservation_report.json` |
+| `layer11` | `2355707` | `xfer` | `afterok:2353960` | `layer11/td_preservation_report.json` |
+
+Queued packed intrinsic eval behind both preservation jobs:
+
+- Job: `2355714`
+- Partition: `normal`
+- Dependency: `afterok:2355706:2355707`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_intrinsics_20260523T124000Z`
+- Model spec:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_intrinsics_20260523T124000Z/model_spec.tsv`
+- Arms:
+  - `retok`
+  - `td_full25_last`
+  - `td_full25_layer11`
+
+This ordering means the full-token eval will only consume a GPU node if TD
+training succeeds and both artifact-preservation checks pass.
+
+## Full-token layer11 artifact complete
+
+`target_layer=11` completed first:
+
+- Completion line: `=== layer complete: 11 ===`
+- Completion timestamp: 2026-05-23 13:57:09 UTC
+- Output dir:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_full25_layers_20260523T092602Z/layer11`
+- Manifest: `completed = True`
+- Elapsed: `16,254.8s`
+- Final training size: `54,303` dataloader steps
+- Manifest counts:
+  - `trained_token_count = 17,377`
+  - `skipped_token_count = 15`
+  - `target_layer = 11`
+  - `snippets_per_token = 25`
+  - `batch_size = 8`
+- Saved model files:
+  - `model.safetensors.index.json`
+  - `model-00001-of-00004.safetensors`
+  - `model-00002-of-00004.safetensors`
+  - `model-00003-of-00004.safetensors`
+  - `model-00004-of-00004.safetensors`
+  - `config.json`, tokenizer files, and generation/special-token configs.
+
+Final layer11 training log excerpts:
+
+- `Total end-to-end time: 16068.92639554094`
+- `saving TD checkpoint: .../retok_td_full25_layers_20260523T092602Z/layer11`
+- `done in 16254.8s`
+
+Overall Slurm job `2353960` remains `RUNNING` because the `last` candidate is
+still training. At 13:58 UTC it was `42,725 / 54,303` (`78.7%`) with ETA
+`1:13:53`.
+
+## Full-token training complete
+
+The packed full-token TD training job completed cleanly:
+
+- Job: `2353960`
+- State: `COMPLETED`
+- Elapsed: `05:45:53`
+- Exit code: `0:0`
+- Restart count observed while running: `0`
+
+Additional `last` progress snapshots after `layer11` had completed:
+
+| UTC time | last progress | last ETA |
+|---|---:|---:|
+| 14:03 | 43,519 / 54,303 (80.1%) | 1:06:42 |
+| 14:08 | 44,302 / 54,303 (81.6%) | 1:04:37 |
+| 14:13 | 45,099 / 54,303 (83.1%) | 0:57:57 |
+| 14:18 | 45,892 / 54,303 (84.5%) | 0:52:19 |
+| 14:23 | 46,689 / 54,303 (86.0%) | 0:47:19 |
+| 14:28 | 47,483 / 54,303 (87.4%) | 0:41:06 |
+| 14:33 | 48,279 / 54,303 (88.9%) | 0:39:44 |
+| 14:38 | 49,074 / 54,303 (90.4%) | 0:32:19 |
+| 14:43 | 49,869 / 54,303 (91.8%) | 0:28:37 |
+| 14:48 | 50,663 / 54,303 (93.3%) | 0:22:41 |
+| 14:53 | 51,459 / 54,303 (94.8%) | 0:17:04 |
+| 14:58 | 52,245 / 54,303 (96.2%) | 0:12:52 |
+| 15:03 | 53,038 / 54,303 (97.7%) | 0:07:52 |
+| 15:08 | 53,830 / 54,303 (99.1%) | 0:02:57 |
+| 15:13 | 54,303 / 54,303 (100.0%) | 0:00:00 |
+
+`last` output dir:
+`/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_full25_layers_20260523T092602Z/last`
+
+`last` artifact state:
+
+- `model.safetensors.index.json` exists.
+- `retok_td_manifest.json` was rewritten with `completed = True`.
+- Manifest counts match `layer11`:
+  - `trained_token_count = 17,377`
+  - `skipped_token_count = 15`
+  - `snippets_per_token = 25`
+
+## Full-token preservation gates passed
+
+Both xfer preservation jobs passed:
+
+| Arm | Job | State | Elapsed | Report |
+|---|---:|---|---:|---|
+| `last` | `2355706` | `COMPLETED` | `00:01:36` | `last/td_preservation_report.json` |
+| `layer11` | `2355707` | `COMPLETED` | `00:01:39` | `layer11/td_preservation_report.json` |
+
+Summary from both preservation reports:
+
+- `trained_token_count = 17,377`
+- `non_embedding_changed_count = 0`
+- `shape_mismatches = []`
+- `dtype_mismatches = []`
+- `xielu_changed = []`
+- `qk_norm_changed = []`
+- `lm_head.weight`:
+  - preserved rows changed: `0`
+  - trained rows changed: `17,377`
+  - trained rows unchanged: `0`
+  - nonfinite trained rows: `0`
+  - zero trained rows: `0`
+- `model.embed_tokens.weight`:
+  - preserved rows changed: `0`
+  - trained rows changed: `17,377`
+  - trained rows unchanged: `0`
+  - nonfinite trained rows: `0`
+  - zero trained rows: `0`
+
+Interpretation: artifact-level preservation passed for both full-token TD
+candidates. xIELU, QK-Norm, all transformer tensors, and all untrained input and
+output rows are byte-exact relative to the ReTok reference. Only the intended
+new-token input/output rows moved.
+
+The packed intrinsic eval job is now dependency-cleared:
+
+- Job: `2355714`
+- State after preservation: `PENDING`
+- Reason: `Priority`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_intrinsics_20260523T124000Z`
+
+## Full-token intrinsic eval complete
+
+Packed intrinsic eval completed cleanly:
+
+- Job: `2355714`
+- State: `COMPLETED`
+- Elapsed: `00:03:15`
+- Exit code: `0:0`
+- Output root:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_intrinsics_20260523T124000Z`
+- Local copy:
+  `../eval/td_full25_intrinsics_20260523T124000Z/`
+
+Heldout Greek metrics:
+
+| Arm | BPC | Delta vs ReTok | NLL/char | tokens/word | STRR |
+|---|---:|---:|---:|---:|---:|
+| `retok` | 2.9503 | 0.0000 | 3.4896 | 1.7352 | 0.4458 |
+| `td_full25_last` | 1.4249 | -1.5254 | 1.6853 | 1.7352 | 0.4458 |
+| `td_full25_layer11` | **1.3846** | **-1.5657** | **1.6376** | 1.7352 | 0.4458 |
+
+New-token diagnostics:
+
+| Arm | D1 mean rank | D1 top1 | D1 top5 | D2 new mass | D4 top1-new | D5 greedy-new | E norm ratio | U norm ratio |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| `retok` | 3868.27 | 0.0065 | 0.0231 | 0.9821 | 0.9992 | 1.0000 | 1.0578 | 1.0140 |
+| `td_full25_last` | 1756.04 | 0.0381 | 0.1596 | 0.8194 | 0.9266 | 0.9980 | 1.0585 | 1.0195 |
+| `td_full25_layer11` | **1617.48** | **0.0415** | **0.1722** | 0.8094 | 0.9153 | 0.9960 | 1.0590 | 1.0195 |
+
+Interpretation:
+
+- Full-token TD is not a marginal improvement over ReTok; it is a large
+  pre-CPT heldout-NLL improvement while preserving the extended tokenizer's
+  compression (`tokens/word` and `STRR` unchanged).
+- `layer11` beats `last` on BPC and D1 rank/top-k, matching the pilot result.
+- D2/D4/D5 move down from near-saturated ReTok use of new IDs, but remain high.
+  That is the desired direction here: less indiscriminate new-token mass while
+  keeping strong new-token availability.
+- New input/output row norm ratios remain close to ReTok and do not show a norm
+  explosion.
+
+Current decision:
+
+- Select `td_full25_layer11` as the only TD checkpoint worth taking to the next
+  gate.
+- Do not spend on a 50/100-snippet TD run before conversion/smoke; the 25-snippet
+  full-token checkpoint already clearly beats ReTok intrinsically.
+- Next gate: R17-preserving HF -> Megatron conversion and roundtrip/load smoke
+  for `td_full25_layer11`.
+
+## TD layer11 R17 conversion and HF roundtrip passed
+
+Before launching the conversion gate, I checked whether it could safely run on
+`xfer` and avoid a GPU allocation. It could not:
+
+| Job | Partition | State | Finding |
+|---:|---|---|---|
+| `2357552` | `xfer` | failed | `sbatch --uenv` did not provide `torch` |
+| `2357556` | `xfer` | failed | explicit `uenv run` path still had no `torch` |
+| `2357561` | `xfer` | failed | `srun` on this site rejected `--uenv=...` |
+
+The conversion gate therefore ran as a bounded `debug` GPU job because it
+requires PyTorch/model loading and includes an HF logit smoke. This was not a
+CPU-only dataset/snippet path.
+
+R17 roundtrip job:
+
+- Job: `2357565`
+- Partition: `debug`
+- State: `COMPLETED`
+- Elapsed: `00:02:06`
+- Exit code: `0:0`
+- Input HF TD checkpoint:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/retok_td_full25_layers_20260523T092602Z/layer11`
+- Raw Megatron output:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_raw`
+- R17-patched Megatron output:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+- HF roundtrip output:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/hf_roundtrip`
+- Verification JSON on Clariden:
+  `/capstor/scratch/cscs/fffoivos/runs/td_r17_roundtrip_2357565/verification.json`
+- Local evidence copy:
+  `../megatron_patches/td_layer11_r17_roundtrip_2357565/`
+
+Verifier result:
+
+| Check | Result |
+|---|---:|
+| `standard_max_abs_diff` | `0.0` |
+| `r17_max_abs_diff` | `0.0` |
+| `xielu_max_abs_diff` | `0.0` |
+| `qk_norm_max_abs_diff` | `0.0` |
+| `standard_changed_over_tol_count` | `0` |
+| `shape_mismatches` | `[]` |
+| `logit_max_abs_diff` | `0.0` |
+| `logit_mean_abs_diff_max` | `0.0` |
+
+All three smoke prompts had exact logit parity and matching top IDs:
+
+- `Η Αθήνα είναι η πρωτεύουσα της`
+- `Ο Όμηρος έγραψε`
+- `The capital of Greece is`
+
+Interpretation:
+
+- The selected TD layer11 checkpoint is now proven safe through the
+  R17-preserving HF -> Megatron -> HF roundtrip.
+- The raw HF -> Megatron route is still not acceptable in general; the safe
+  production artifact is the patched Megatron checkpoint above.
+- Next gate: a very short Megatron load/train smoke from
+  `megatron_tp2_r17patched`, with saves disabled or pushed beyond the smoke,
+  before any longer CPT-scale run.
+
+## TD layer11 Megatron load/train smoke setup
+
+Added `../bakeoff_training/submit_td_layer11_smoke.sh` so the next gate is a
+reproducible bounded run, not a hand-typed one-off. It reuses
+`bakeoff_train.sbatch` and only changes:
+
+- `ARM=retok`, so the extended tokenizer and extended-tokenized Megatron data
+  are used.
+- `INIT_CKPT`, pointed at the selected R17-patched TD checkpoint:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+- `EXIT_INTERVAL=5`
+- `SAVE_INTERVAL=999999`
+- `EVAL_INTERVAL=999999`
+- `TIME_LIMIT=01:00:00`
+
+This gate is intentionally just a load/first-forward/backward/optimizer-step
+smoke. It should not produce a CPT checkpoint or consume a long training window.
+
+Launch:
+
+- Job: `2357596`
+- Partition: `normal`
+- Output directory:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/smoke_td_full25_layer11_20260523T162614Z`
+- Settings:
+  - `TIME_LIMIT=01:00:00`
+  - `EXIT_INTERVAL=5`
+  - `SAVE_INTERVAL=999999`
+  - `EVAL_INTERVAL=999999`
+  - `ARM=retok`
+  - `INIT_CKPT=/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+
+## TD layer11 two-node efficiency smoke failed
+
+Smoke result:
+
+- Job: `2357684`
+- State: `FAILED`
+- Elapsed: `00:02:24`
+- Exit code: `1:0`
+- Nodes: `nid006031`, `nid007615`
+- Local evidence copy:
+  `../bakeoff_training/smoke_td_layer11_2node_2357684/`
+
+What passed before failure:
+
+- The selected R17-patched checkpoint loaded successfully.
+- The extended tokenizer/data were selected through `ARM=retok`.
+- The `torchrun` world size was `8`.
+- xIELU optimizer audit passed on all eight ranks with `missing=0`.
+
+Failure:
+
+- The job failed before iteration 1 during a data-parallel collective.
+- Root error in stderr:
+  `NET/OFI Request ... Error: 16 (NO_SPACE)`.
+- PyTorch surfaced it as:
+  `NCCL Error 2: unhandled system error`.
+
+Decision:
+
+- Do not spend the 2B TD arm on the two-node path right now.
+- Use the reliable one-node path.
+- `normal` partition has a `12:00:00` walltime cap, while one-node smoke
+  throughput implies about 17.5h for 2B. Therefore the safe 2B launch shape is
+  a one-node initial job plus an automatic resume job from the same checkpoint
+  directory.
+
+## TD layer11 2B chained training launched
+
+Launched the decision-useful 2B TD arm on the proven one-node path:
+
+- Output directory:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z`
+- Initial job: `2357704`
+- Resume job: `2357705`
+- Dependency: `afterany:2357704`
+- Partition: `normal`
+- Nodes: `1`
+- GPUs: `4`
+- Launch mode: `slurm`
+- Time limit per job: `12:00:00`
+- Save interval: `65`
+- Eval interval: `999999`
+- Chain resume: `1`
+- Init checkpoint:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+
+Reason for chained shape: the partition limit is 12h, and one-node smoke
+throughput gives an estimated 17-18h for 2B tokens. The first job should save
+periodic checkpoints; the second job should resume from
+`$OUTPUT_DIR/checkpoints`.
+
+Initial live health:
+
+- Job `2357704` started on `nid006069`.
+- Resume job `2357705` remains pending with `Dependency`, as intended.
+- Checkpoint load succeeded from the TD R17-patched init checkpoint.
+- Dataset build succeeded with the extended tokenizer/data prefix.
+- xIELU optimizer audit passed on all four ranks with `missing=0`.
+- First iteration completed:
+  - timestamp: `2026-05-23 18:55:41`
+  - iteration: `1/476`
+  - consumed tokens: `0.004B`
+  - loss: `8.928761`
+  - grad norm: `153.597`
+  - tokens/sec/GPU: `7553.6`
+  - skipped iterations: `0`
+  - NaN iterations: `0`
+  - max allocated memory after iter 1: `61378.95 MB`
+
+Interpretation: the decision-useful 2B TD arm is live and healthy on the proven
+one-node path. Expected remaining walltime from observed per-iteration timing is
+still longer than the 12h partition cap, so the dependent resume job is needed.
+
+Follow-up health:
+
+- At Slurm elapsed `00:06:18`, iteration 2 completed:
+  - timestamp: `2026-05-23 18:57:55`
+  - iteration: `2/476`
+  - consumed tokens: `0.008B`
+  - loss: `9.013231`
+  - grad norm: `102.030`
+  - tokens/sec/GPU: `7852.9`
+  - skipped iterations: `0`
+  - NaN iterations: `0`
+
+At launch there were no other visible Slurm jobs for `fffoivos`.
+
+## TD layer11 Megatron load/train smoke passed
+
+Smoke result:
+
+- Job: `2357596`
+- State: `COMPLETED`
+- Elapsed: `00:13:19`
+- Exit code: `0:0`
+- Node: `nid007017`
+- Local evidence copy:
+  `../bakeoff_training/smoke_td_layer11_2357596/`
+
+The smoke loaded the selected R17-patched checkpoint, selected the extended
+tokenizer/data through `ARM=retok`, built the dataset, passed the first
+forward/backward/optimizer path, and exited at iteration 5.
+
+Iteration metrics:
+
+| Iter | Loss | Grad norm | Tokens/sec/GPU | Skipped | NaN |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 8.928761 | 153.403 | 7546.1 | 0 | 0 |
+| 2 | 9.013231 | 101.992 | 7867.7 | 0 | 0 |
+| 3 | 8.947750 | 105.898 | 7907.7 | 0 | 0 |
+| 4 | 8.990582 | 106.622 | 7905.1 | 0 | 0 |
+| 5 | 8.853868 | 173.612 | 7849.6 | 0 | 0 |
+
+Other health signals:
+
+- Checkpoint load succeeded from:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+- xIELU optimizer audit passed on all four ranks:
+  `missing=0`, `dp_optimizer_alpha_params=64`.
+- Rank 0/1 memory after iteration 1:
+  max allocated `61378.95 MB`, max reserved `61662.0 MB`.
+- The warnings were non-fatal NCCL/barrier/deprecation/shutdown warnings; no
+  OOM, no NaN, and no skipped iterations appeared.
+
+Unexpected save behavior and cleanup:
+
+- `EXIT_INTERVAL=5` caused Megatron to save at exit despite
+  `SAVE_INTERVAL=999999`, because the training loop saves on exit whenever
+  `args.save` is set.
+- The smoke checkpoint was `138G`.
+- I captured `checkpoint_listing_before_cleanup.txt` locally, then removed the
+  smoke checkpoint directory from Capstor. The remote smoke run directory is
+  now `64K`.
+- I added a `DISABLE_SAVE=1` path for future TD smokes so this wrapper does not
+  produce another accidental large checkpoint. Normal training still defaults
+  to checkpointing enabled.
+
+## TD layer11 two-node efficiency smoke launched
+
+Reason: the one-node smoke throughput is roughly `7.85k-7.91k tokens/sec/GPU`,
+which implies about 17.5 hours for the 2B TD arm on one 4-GPU node. Before
+launching the decision-useful 2B arm, I started a bounded two-node smoke to
+verify that the existing `torchrun` multi-node path works for this exact TD
+checkpoint/data stack.
+
+Launch:
+
+- Job: `2357684`
+- Partition: `normal`
+- Nodes: `2`
+- GPUs: `8`
+- Launch mode: `torchrun`
+- Output directory:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/smoke_td_full25_layer11_2node_20260523T164424Z`
+- Settings:
+  - `EXIT_INTERVAL=2`
+  - `DISABLE_SAVE=1`
+  - `SAVE_INTERVAL=999999`
+  - `EVAL_INTERVAL=999999`
+  - `ARM=retok`
+  - `INIT_CKPT=/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+
+## Continuation note: eval handoff and live 2B status
+
+Timestamp: `2026-05-23T17:04:06Z`
+
+I resumed from the TD 2B run and explicitly recorded the handoff steps so the
+run remains reconstructable.
+
+Current live training state:
+
+- Active training job: `2357704` (`td_l11_2b`)
+- Resume job: `2357705` (`td_l11_2b_resume`), pending on `afterany:2357704`
+- Run directory:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z`
+- Init checkpoint:
+  `/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched`
+- Status check at Slurm elapsed `00:15:13`: job `2357704` was running on
+  `nid006069`.
+- Latest observed iteration: `6/476`
+  - timestamp: `2026-05-23 19:06:46`
+  - consumed tokens: `0.025B`
+  - loss: `8.708739`
+  - grad norm: `159.085`
+  - tokens/sec/GPU: `7890.4`
+  - skipped iterations: `0`
+  - NaN iterations: `0`
+
+Two-node smoke decision:
+
+- Two-node smoke job `2357684` failed before iteration 1 with
+  `NET/OFI ... Error: 16 (NO_SPACE)` and `NCCL Error 2`.
+- I did not switch the production TD 2B run to the two-node path. The active
+  run remains on the reliable one-node path.
+
+Eval handoff changes prepared locally:
+
+- `eval/convert_bakeoff_checkpoint_to_hf.sbatch` now accepts
+  `ARM=td_layer11` and defaults that arm to the TD layer11 R17-roundtripped HF
+  tokenizer/config directory.
+- Added `eval/submit_td_checkpoint_eval.sh`, a direct-path submitter for the
+  selected TD run's checkpoints.
+- The submitter preflights `EVAL_JSONL` before any `sbatch` submission when
+  `SUBMIT_INTRINSIC=1`, so a misconfigured intrinsic run does not half-submit.
+- Added `eval/watch_and_submit_td_checkpoint_eval.sh`, a direct-path watcher
+  that submits conversion/eval once both the checkpoint directory and
+  `latest_checkpointed_iteration.txt` confirm the target iteration is ready.
+- Updated `eval/EVAL_RECIPE.md` with the TD direct-path eval handoff.
+- Verified the held-out intrinsic slice still exists at:
+  `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/heldout/cpt_greek_heldout_500_20260522.jsonl`
+  with 500 docs and size `33M`.
+
+Checkpoint watchers launched:
+
+| Iter | Scope | Intrinsic | PID | Watch log |
+|---:|---|---:|---:|---|
+| 65 | `greek_only` | 0 | 198851 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter65_greek_only.log` |
+| 130 | `full` | 1 | 198853 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter130_full.log` |
+| 260 | `full` | 1 | 198854 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter260_full.log` |
+| 390 | `full` | 1 | 198855 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter390_full.log` |
+| 455 | `full` | 1 | 198856 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter455_full.log` |
+| 476 | `full` | 1 | 198857 | `/capstor/scratch/cscs/fffoivos/runs/eval/watch_td_l11_iter476_full.log` |
+
+All six watchers wrote their initial `waiting for iter_...` line at
+`2026-05-23T17:08:41Z`.
+
+Watcher durability correction:
+
+- The first watcher launch used `nohup` from the SSH login session. It wrote the
+  state directories and initial logs, but the processes did not persist after
+  the SSH session closed.
+- A direct `sbatch --wrap` attempt with a 36h xfer walltime failed because the
+  requested time limit exceeded the partition limit.
+- I added `eval/watch_td_checkpoint_evals.sbatch`: one CPU, no GPU,
+  `partition=xfer`, `time=11:50:00`, running the same watcher set under Slurm.
+- Durable watcher jobs:
+  - `2357765` running on `xfer` (`td_l11_watch`), started at
+    `2026-05-23T17:11:43Z` on `nid001154`.
+  - `2357766` pending on `afterany:2357765`, same wrapper, to cover the second
+    half of the chained 2B training window.
+- The wrapper reuses per-checkpoint `submitted` stamps, so the second watcher
+  window can safely repeat all six targets without duplicate eval submissions.
+
+Important resource guard:
+
+- Checkpoint conversion remains a CPU-only `xfer` Slurm job through
+  `slurm_cpu_only_guard.sh`.
+- Downstream lm-eval and intrinsic metrics remain GPU jobs, as intended.
+
+## Health pulse: pre-checkpoint TD 2B run
+
+Timestamp: `2026-05-23T17:26:45Z`
+
+Status:
+
+- Training job `2357704` still running on `normal`, node `nid006069`, elapsed
+  `00:34:33`.
+- Resume job `2357705` still pending on dependency, as intended.
+- Durable watcher job `2357765` still running on `xfer`, node `nid001154`,
+  elapsed `00:14:52`.
+- Second watcher job `2357766` still pending on `afterany:2357765`, as
+  intended.
+
+Latest training line:
+
+- Iteration: `14/476`
+- Timestamp in training log: `2026-05-23 19:24:29`
+- Consumed tokens: `0.059B`
+- Loss: `6.599847`
+- Grad norm: `23.327`
+- Tokens/sec/GPU: `7893.5`
+- Skipped iterations: `0`
+- NaN iterations: `0`
+
+Checkpoint/eval state:
+
+- No checkpoint files yet under the TD 2B checkpoint root; this is expected
+  because the first save target is iteration `65`.
+- The iter-65 watcher is polling every 300 seconds and still waiting for
+  `checkpoints/iter_0000065`.
+
+## Checkpoint 65 and eval canary handoff
+
+Timestamp: `2026-05-23T19:24Z`
+
+Training/checkpoint state:
+
+- Training job `2357704` reached iteration `65/476`.
+- Iteration 65 metrics:
+  - training-log timestamp: `2026-05-23 21:17:27`
+  - consumed tokens: `0.273B`
+  - loss: `3.919139`
+  - grad norm: `5.525`
+  - tokens/sec/GPU: `7888.5`
+  - skipped iterations: `0`
+  - NaN iterations: `0`
+- Checkpoint save completed:
+  - checkpoint dir:
+    `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z/checkpoints/iter_0000065`
+  - tracker:
+    `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z/checkpoints/latest_checkpointed_iteration.txt`
+    contains `65`.
+- Training continued to iteration `66/476` after the async save.
+
+Watcher/eval canary:
+
+- Durable watcher `2357765` detected checkpoint 65 and submitted:
+  - conversion job `2358857` (`tohf_td_l11_65`)
+  - dependent Greek-only eval job `2358858`
+
+Canary failure and correction:
+
+- Conversion job `2358857` failed in `00:00:04` with exit `127:0`.
+- Root cause from stderr:
+  `uenv: command not found`.
+- A fresh xfer probe job `2358876` confirmed that `xfer` node `nid001154` has
+  no `uenv` binary at either `command -v uenv` or `/usr/bin/uenv`, matching the
+  earlier TD notes for conversion gates.
+- I cancelled the dead dependent eval job `2358858`.
+- I changed `eval/convert_bakeoff_checkpoint_to_hf.sbatch` from a forced
+  CPU-only `xfer` job to a bounded one-GPU `debug` job. This matches the earlier
+  R17/TD conversion gate: Megatron checkpoint conversion imports PyTorch and
+  needs the PyTorch uenv, so it cannot run on xfer on this site.
+- I removed `eval/convert_bakeoff_checkpoint_to_hf.sbatch` from
+  `check_cpu_only_slurm.sh`'s CPU-only audit list. Dataset/build/preprocessing
+  jobs remain guarded for `xfer`; checkpoint watchers also remain CPU-only
+  `xfer` jobs.
+
+Next action:
+
+- Sync the corrected conversion script/docs to Clariden.
+- Resubmit the iter-65 conversion/eval chain and verify the Greek-only canary
+  starts from the corrected conversion job.
+
+## Checkpoint 65 conversion recovery passed
+
+Timestamp: `2026-05-23T19:28:04Z`
+
+Local changes committed and synced:
+
+- Commit: `8428b6d` (`Fix TD checkpoint conversion partition`)
+- Changed `eval/convert_bakeoff_checkpoint_to_hf.sbatch` to run on
+  `debug` with one GPU and the PyTorch uenv.
+- Removed `eval/convert_bakeoff_checkpoint_to_hf.sbatch` from the CPU-only
+  static audit list.
+- Updated `eval/EVAL_RECIPE.md` to distinguish:
+  - true CPU-only dataset/build/watch jobs -> `xfer`
+  - Megatron/PyTorch checkpoint conversion -> bounded one-GPU `debug`
+
+Validation before resubmit:
+
+- Local:
+  - `bash -n eval/convert_bakeoff_checkpoint_to_hf.sbatch`
+  - `bash -n check_cpu_only_slurm.sh`
+  - `./check_cpu_only_slurm.sh`
+- Remote on Clariden mirror:
+  - `bash -n eval/convert_bakeoff_checkpoint_to_hf.sbatch`
+  - `bash -n check_cpu_only_slurm.sh`
+  - `./check_cpu_only_slurm.sh`
+
+All static checks passed. The CPU-only audit covers 10 true CPU-only jobs.
+
+Resubmitted iter-65 canary:
+
+- Command shape:
+  `RUN_TAG=td_full25_layer11_2b_20260523T165038Z OVERWRITE=1 bash submit_td_checkpoint_eval.sh 65 greek_only`
+- Conversion job: `2358891` (`tohf_td_l11_65`)
+- Greek-only eval job: `2358892` (`eval_td_l11_65_greek_only`)
+
+Conversion result:
+
+- Job `2358891` completed successfully.
+- Partition: `debug`
+- Elapsed: `00:01:15`
+- Exit code: `0:0`
+- HF checkpoint:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000065_hf`
+- HF artifacts include:
+  - `config.json`
+  - `tokenizer.json`
+  - `model.safetensors.index.json`
+  - `model-00001-of-00004.safetensors` through `model-00004-of-00004.safetensors`
+  - `bakeoff_conversion_metadata.json`
+
+Eval state:
+
+- Job `2358892` started on `normal`, node `nid007637`.
+- This is the Greek-only downstream canary for checkpoint 65.
+
+## Checkpoint 65 Greek-only canary passed
+
+Timestamp: `2026-05-23T20:02:06Z`
+
+Training state at refresh:
+
+- Main TD 2B job `2357704` is still running on `normal`, node `nid006069`.
+- Resume job `2357705` remains pending behind `2357704`.
+- Durable checkpoint watcher `2357765` is still running on `xfer`, node
+  `nid001154`.
+- Backup watcher `2357766` remains pending behind `2357765`.
+- Latest parsed training line: iteration `84/476`, consumed tokens `0.352B`,
+  timestamp `2026-05-23 21:59:58`, loss `3.593952`, throughput
+  `7876.2` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint tracker remains at `65`; next planned checkpoint is iteration
+  `130`.
+
+Greek-only canary result:
+
+- Eval job `2358892` completed successfully.
+- Partition/node: `normal` / `nid007637`
+- Elapsed: `00:22:15`
+- Exit code: `0:0`
+- Result JSON:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000065_greek_only/results_2026-05-23T21-49-04.839736.json`
+
+Headline metrics from checkpoint 65:
+
+- `global_mmlu_full_el` accuracy: `0.3717419171`
+- `include_base_44_greek_few_shot_en` accuracy: `0.3713768116`
+- `xnli_el` accuracy: `0.3903614458`
+- `arc_challenge_mt_el` normalized accuracy: `0.3233788396`
+- `belebele_ell_Grek` accuracy: `0.4877777778`
+- `global_piqa_completions_ell_grek` normalized accuracy: `0.53`
+- `xquad_el` exact match: `0.1537815126`
+- `xquad_el` F1: `0.2992209373`
+
+Interpretation:
+
+- The corrected conversion/eval path works end to end from a live Megatron
+  checkpoint to HF and then lm-eval.
+- Checkpoint 65 is an early canary, not a final recipe-selection checkpoint.
+  It is useful because it proves the model is not collapsed after TD init and
+  early CPT, and it validates the automation before the more decision-relevant
+  checkpoint 130 full eval.
+
+Next action:
+
+- Keep monitoring the live training job and watcher.
+- At iteration `130`, verify that the watcher submits the full downstream eval
+  and intrinsic eval chain with the corrected conversion job.
+
+## Runtime expectation after checkpoint 65
+
+Timestamp: `2026-05-23T20:07:30Z`
+
+Live training refresh:
+
+- Main TD 2B job `2357704` is still running.
+- Latest parsed line: iteration `86/476`, consumed tokens `0.361B`, timestamp
+  `2026-05-23 22:04:25`, loss `3.559302`, throughput `7874.4`
+  tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint tracker is still `65`.
+
+Runtime note:
+
+- The live Megatron ETA at iteration 86 is still about `14:25:33`, while job
+  `2357704` has already used about `3:12` of its `12:00` allocation.
+- Therefore the first allocation is expected to hit walltime before iteration
+  `476`.
+- This is not currently a failure condition because resume job `2357705` is
+  already pending with the dependency chain. The next thing to watch later is
+  that the handoff resumes from the latest completed checkpoint cleanly.
+
+## Checkpoint 130 saved and full eval chain submitted
+
+Timestamp: `2026-05-23T21:48:33Z`
+
+Training state:
+
+- Main TD 2B job `2357704` is still running on `normal`, node `nid006069`.
+- Resume job `2357705` remains pending behind `2357704`.
+- Durable watcher `2357765` is still running on `xfer`, node `nid001154`.
+- Latest parsed training line after the checkpoint: iteration `132/476`,
+  consumed tokens `0.554B`, timestamp `2026-05-23 23:46:27`, loss
+  `3.140112`, throughput `7868.7` tokens/sec/GPU, skipped iterations `0`,
+  NaN iterations `0`.
+
+Checkpoint state:
+
+- Iteration `130` was saved successfully.
+- Checkpoint tracker now reads `130`.
+- Checkpoint dirs currently present:
+  - `iter_0000065`
+  - `iter_0000130`
+- Save log included:
+  - `saving checkpoint at iteration     130`
+  - `scheduled an async checkpoint save at iteration     130`
+  - `successfully saved checkpoint from iteration     130`
+
+Watcher/eval state:
+
+- Watcher `watch_td_l11_iter130_full.log` saw the checkpoint directory before
+  the tracker updated, waited correctly, then submitted once tracker reached
+  `130`.
+- Submission log:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z_watch_iter_0000130_full/submit.log`
+- Submitted jobs:
+  - conversion: `2360120` (`tohf_td_l11_130`)
+  - full lm-eval: `2360121` (`eval_td_l11_130_full`)
+  - intrinsic BPC: `2360122` (`bpc_td_l11_130`)
+  - new-token diagnostics: `2360123` (`diag_td_l11_130`)
+
+Immediate verification:
+
+- Conversion job `2360120` completed successfully on `debug`.
+- Elapsed: `00:01:13`
+- Exit code: `0:0`
+- HF output:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000130_hf`
+- The HF output contains the expected config/tokenizer/index metadata.
+- At this timestamp, full lm-eval job `2360121` and intrinsic BPC job
+  `2360122` are running; diagnostics job `2360123` is pending for resources.
+
+Next action:
+
+- Wait for jobs `2360121`, `2360122`, and `2360123`.
+- Parse checkpoint-130 downstream metrics and intrinsic/new-token metrics.
+- Log and commit the checkpoint-130 results once they are complete.
+
+## Checkpoint 130 intrinsic and new-token diagnostics passed
+
+Timestamp: `2026-05-23T21:56:40Z`
+
+Completed jobs:
+
+- Intrinsic BPC job `2360122` completed successfully.
+  - Elapsed: `00:01:39`
+  - Exit code: `0:0`
+- New-token diagnostics job `2360123` completed successfully.
+  - Elapsed: `00:02:19`
+  - Exit code: `0:0`
+- Full downstream lm-eval job `2360121` is still running at this timestamp.
+
+Artifacts:
+
+- Tokenizer-fair intrinsic metrics:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000130_tokenizer_fair_metrics.json`
+- New-token diagnostics:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000130_new_token_diagnostics.json`
+
+Intrinsic metrics on the 500-doc Greek heldout slice:
+
+- `n_docs`: `500`
+- `n_tokens`: `805972`
+- `chars_per_token`: `3.973178969`
+- `tokens_per_word`: `1.735161272`
+- `bpc_bits_per_byte`: `0.6531070974`
+- `nll_per_token`: `3.069195861`
+- STRR single-token word rate: `0.4458156518`
+
+Per-register `bpc_bits_per_byte`:
+
+- `academic`: `0.5755798070`
+- `dialogue_textbooks`: `0.8630783619`
+- `hplt_clean60`: `0.6225848939`
+- `legal_civic`: `0.4718179034`
+
+Truncation caveat:
+
+- `124/500` docs were truncated at the 4096-token scoring prefix.
+- `fraction_truncated`: `0.248`
+- This is acceptable as a same-slice checkpoint trend metric, but the absolute
+  heldout BPC should be read with the script's caveat: char/byte/word counts
+  are computed on the scored prefix, not the full text.
+
+New-token diagnostics:
+
+- New-token id range requested/applied: `17408` rows, base vocab `131072`,
+  extended vocab `148480`.
+- Input embedding norm ratio, new/existing mean: `1.0589042902`.
+- Output embedding norm ratio, new/existing mean: `1.0192596912`.
+- New-input-row cosine sample:
+  - mean off-diagonal: `0.0813974366`
+  - p95 off-diagonal: `0.1582076550`
+  - p99 off-diagonal: `0.3888086677`
+- New-input-row effective-rank sample:
+  - participation ratio: `324.5376586914`
+  - rank at 99% variance: `394/500`
+- Forward diagnostic over heldout positions:
+  - positions total: `805472`
+  - new-target positions: `276332`
+  - average probability mass on new-token vocab per position: `0.3400108291`
+  - new-target rank top-1: `0.2749410130`
+  - new-target rank top-5: `0.4391746160`
+  - new-target rank top-10: `0.5040169072`
+  - new-target rank top-50: `0.6660864467`
+  - top-1 prediction at new-target positions chose a new token `51.11%` of
+    the time.
+- Greedy generation sanity sample:
+  - prompts: `5`
+  - generated tokens: `500`
+  - generated new-token count: `103`
+  - utilization rate: `0.206`
+
+Interpretation:
+
+- Intrinsics and diagnostics do not show a new-token collapse at checkpoint
+  `130`.
+- New rows remain norm-compatible with existing rows, have non-degenerate
+  sampled rank, and are being used by both forward scoring and greedy
+  generation.
+- The next decision-useful evidence is the still-running full downstream eval
+  for checkpoint `130`.
+
+## Checkpoint 130 full downstream eval passed
+
+Timestamp: `2026-05-23T22:38:22Z`
+
+Completed job:
+
+- Full lm-eval job `2360121` completed successfully.
+  - Elapsed: `00:46:45`
+  - Exit code: `0:0`
+
+Artifact paths:
+
+- Remote result JSON:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000130_full/results_2026-05-24T00-34-38.581138.json`
+- Remote run metadata:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000130_full/run_metadata.json`
+- Local compact copies:
+  - `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000130_results.json`
+  - `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000130_run_metadata.json`
+  - `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000130_tokenizer_fair_metrics.json`
+  - `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000130_new_token_diagnostics.json`
+  - `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000130_digest.md`
+
+Headline full-eval metrics:
+
+- `arc_challenge` acc_norm: `0.5537542662`
+- `arc_easy` acc_norm: `0.8000841751`
+- `hellaswag` acc_norm: `0.7598088030`
+- `winogrande` acc: `0.6850828729`
+- `piqa` acc_norm: `0.7997823721`
+- `mmlu` acc: `0.5484973650`
+- `global_mmlu` acc: `0.4703933786`
+- `xnli` acc: `0.4139759036`
+- `xcopa` acc: `0.6310909091`
+- `arc_challenge_mt_el` acc_norm: `0.3515358362`
+- `belebele_ell_Grek` acc_norm: `0.5`
+- `xnli_el` acc: `0.3791164659`
+- `xquad_el` exact_match: `0.1781512605`
+- `xquad_el` F1: `0.3713167220`
+- `global_mmlu_full_el` acc: `0.3847742487`
+- `include_base_44_greek_few_shot_en` acc: `0.3858695652`
+- `global_piqa_completions_ell_grek` acc_norm: `0.53`
+
+Greek checkpoint-65 to checkpoint-130 deltas:
+
+- `arc_challenge_mt_el` acc_norm: `+0.0281569966`
+- `arc_challenge_mt_el` raw acc: `+0.0418088737`
+- `belebele_ell_Grek` acc_norm: `+0.0122222222`
+- `global_mmlu_full_el` acc: `+0.0130323316`
+- `include_base_44_greek_few_shot_en` acc: `+0.0144927536`
+- `global_piqa_completions_ell_grek` acc_norm: `0.0`
+- `global_piqa_completions_ell_grek` raw acc: `-0.02`
+- `xnli_el` acc: `-0.0112449799`
+- `xquad_el` exact_match: `+0.0243697479`
+- `xquad_el` F1: `+0.0720957847`
+
+Training state after eval completion:
+
+- Main TD 2B job `2357704` is still running.
+- Latest parsed training line: iteration `154/476`, consumed tokens `0.646B`,
+  timestamp `2026-05-24 00:35:15`, loss `3.060735`, throughput `7874.1`
+  tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint tracker remains `130`; next scheduled full checkpoint/eval gate is
+  `260`.
+
+Interpretation:
+
+- Checkpoint 130 gives the first full TD downstream snapshot and the early
+  Greek trend is mostly positive relative to the checkpoint-65 canary.
+- It is still too early to choose TD for production solely from this point:
+  checkpoint `260`, later full checkpoints, and final comparison to the
+  earlier Vanilla/ReTok/Centroid bakeoff are still required.
+
+## Intermediate checkpoint 195 saved
+
+Timestamp: `2026-05-24T00:45:37Z`
+
+Why this matters:
+
+- Checkpoint `195` is not a scheduled downstream-eval gate, but it is the newest
+  durable resume point for the live TD 2B run.
+- If the first 12h allocation hits walltime before the final iteration, the
+  queued resume job should be able to resume from this or a later tracker value.
+
+Training state at refresh:
+
+- Main TD 2B job `2357704` is still running.
+- Resume job `2357705` remains pending.
+- Durable watcher `2357765` is still running on `xfer`.
+- Latest parsed training line: iteration `212/476`, consumed tokens `0.889B`,
+  timestamp `2026-05-24 02:43:57`, loss `2.874373`, throughput `7879.1`
+  tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Tracker now reads `195`.
+- Checkpoint dirs currently present:
+  - `iter_0000065`
+  - `iter_0000130`
+  - `iter_0000195`
+
+Checkpoint 195 save evidence:
+
+- `saving checkpoint at iteration     195`
+- `scheduled an async checkpoint save at iteration     195`
+- `successfully saved checkpoint from iteration     195`
+- Directory:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z/checkpoints/iter_0000195`
+
+Next action:
+
+- Continue monitoring toward scheduled full eval gate `260`.
+
+## Checkpoint 260 saved and full eval chain submitted
+
+Timestamp: `2026-05-24T02:36:40Z`
+
+Training state:
+
+- Main TD 2B job `2357704` is still running.
+- Resume job `2357705` remains pending.
+- Durable watcher `2357765` is still running on `xfer`.
+- Latest parsed training line after the checkpoint: iteration `262/476`,
+  consumed tokens `1.099B`, timestamp `2026-05-24 04:34:53`, loss
+  `2.661077`, throughput `7877.0` tokens/sec/GPU, skipped iterations `0`,
+  NaN iterations `0`.
+
+Checkpoint state:
+
+- Iteration `260` was saved successfully.
+- Checkpoint tracker now reads `260`.
+- Checkpoint dirs currently present:
+  - `iter_0000065`
+  - `iter_0000130`
+  - `iter_0000195`
+  - `iter_0000260`
+- Save log included:
+  - `saving checkpoint at iteration     260`
+  - `scheduled an async checkpoint save at iteration     260`
+  - `successfully saved checkpoint from iteration     260`
+
+Watcher/eval state:
+
+- Watcher `watch_td_l11_iter260_full.log` submitted once tracker reached
+  `260`.
+- Submission log:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z_watch_iter_0000260_full/submit.log`
+- Submitted jobs:
+  - conversion: `2362712` (`tohf_td_l11_260`)
+  - full lm-eval: `2362713` (`eval_td_l11_260_full`)
+  - intrinsic BPC: `2362714` (`bpc_td_l11_260`)
+  - new-token diagnostics: `2362715` (`diag_td_l11_260`)
+
+Immediate verification:
+
+- Conversion job `2362712` completed successfully on `debug`.
+- Elapsed: `00:01:10`
+- Exit code: `0:0`
+- HF output:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000260_hf`
+- At this timestamp, full lm-eval job `2362713`, intrinsic BPC job `2362714`,
+  and diagnostics job `2362715` are running.
+
+Next action:
+
+- Wait for jobs `2362713`, `2362714`, and `2362715`.
+- Parse checkpoint-260 downstream metrics and intrinsic/new-token metrics.
+- Log and commit the checkpoint-260 results once they are complete.
+
+## Checkpoint 260 intrinsic and new-token diagnostics passed
+
+Timestamp: `2026-05-24T02:42:20Z`
+
+Completed jobs:
+
+- Intrinsic BPC job `2362714` completed successfully.
+  - Elapsed: `00:01:42`
+  - Exit code: `0:0`
+- New-token diagnostics job `2362715` completed successfully.
+  - Elapsed: `00:02:19`
+  - Exit code: `0:0`
+- Full downstream lm-eval job `2362713` is still running at this timestamp.
+
+Artifacts:
+
+- Tokenizer-fair intrinsic metrics:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000260_tokenizer_fair_metrics.json`
+- New-token diagnostics:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000260_new_token_diagnostics.json`
+
+Intrinsic metrics on the 500-doc Greek heldout slice:
+
+- `n_docs`: `500`
+- `n_tokens`: `805972`
+- `chars_per_token`: `3.973178969`
+- `tokens_per_word`: `1.735161272`
+- `bpc_bits_per_byte`: `0.5670351472`
+- `nll_per_token`: `2.664711398`
+- STRR single-token word rate: `0.4458156518`
+
+Per-register `bpc_bits_per_byte`:
+
+- `academic`: `0.4819600870`
+- `dialogue_textbooks`: `0.7560400097`
+- `hplt_clean60`: `0.5585149641`
+- `legal_civic`: `0.3886621322`
+
+Truncation caveat:
+
+- Same as checkpoint `130`: `124/500` docs were truncated at the 4096-token
+  scoring prefix, so absolute BPC should be treated as a same-slice trend
+  metric rather than full-document perplexity.
+
+New-token diagnostics:
+
+- New-token id range applied: `17408` rows, base vocab `131072`, extended vocab
+  `148480`.
+- Input embedding norm ratio, new/existing mean: `1.0588757992`.
+- Output embedding norm ratio, new/existing mean: `1.0193959475`.
+- New-input-row cosine sample:
+  - mean off-diagonal: `0.0813098699`
+  - p95 off-diagonal: `0.1580153704`
+- New-input-row effective-rank sample:
+  - participation ratio: `324.6900024414`
+  - rank at 99% variance: `395/500`
+- Forward diagnostic over heldout positions:
+  - positions total: `805472`
+  - new-target positions: `276332`
+  - average probability mass on new-token vocab per position: `0.3411220586`
+  - new-target rank top-1: `0.3517146042`
+  - new-target rank top-5: `0.5200483476`
+  - new-target rank top-10: `0.5848797823`
+  - new-target rank top-50: `0.7358648293`
+  - top-1 prediction at new-target positions chose a new token `59.52%` of the
+    time.
+- Greedy generation sanity sample:
+  - prompts: `5`
+  - generated tokens: `500`
+  - generated new-token count: `98`
+  - utilization rate: `0.196`
+
+Trend from checkpoint 130 to 260:
+
+- Heldout BPC improved from `0.6531070974` to `0.5670351472`.
+- Heldout NLL/token improved from `3.069195861` to `2.664711398`.
+- New-target top-1 improved from `0.2749410130` to `0.3517146042`.
+- New-target top-10 improved from `0.5040169072` to `0.5848797823`.
+- Average probability mass on the new-token vocab is stable:
+  `0.3400108291` -> `0.3411220586`.
+- New-row input/output norm ratios and sampled rank are stable.
+
+Interpretation:
+
+- Checkpoint `260` intrinsics show meaningful heldout improvement and stronger
+  new-token prediction without evidence of new-row collapse.
+- The remaining checkpoint-260 evidence needed is the still-running full
+  downstream eval.
+
+## Live ETA and walltime handoff check
+
+Timestamp: `2026-05-24T03:17:12Z`
+
+Training state:
+
+- Main TD 2B job `2357704` is still running on `normal`.
+- Resume job `2357705` is pending on dependency.
+- Durable watcher `2357765` is still running on `xfer`; backup watcher
+  `2357766` remains pending.
+- Latest parsed training line: iteration `281/476`, consumed tokens `1.179B`,
+  timestamp `2026-05-24 05:17:02`, loss `2.658228`, throughput `7886.9`
+  tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint tracker still reads `260`; checkpoint dirs present are
+  `65`, `130`, `195`, and `260`.
+
+Checkpoint-260 full eval state:
+
+- Full downstream eval job `2362713` is still running.
+- Elapsed at poll: `00:41:25`.
+- Output directory currently has `run_metadata.json`; the final
+  `results_*.json` has not appeared yet.
+- Eval stdout has completed batch-size probing and selected batch size `40`.
+
+Walltime / resume safety:
+
+- Current allocation elapsed: `10:25:11` of `12:00:00`, with about `1:35`
+  remaining at the poll.
+- At the observed iteration cadence, the allocation will not finish all
+  `476` iterations; it should need the pending resume job.
+- The Slurm script has `#SBATCH --signal=SIGUSR2@600`, and the Megatron
+  training code path for `--exit-signal-handler` was inspected on Clariden:
+  on receiving SIGUSR2, it calls `save_checkpoint_and_time(...)` before
+  exiting. This means the expected walltime handoff should save a fresh
+  checkpoint near the end of the allocation rather than forcing the resume to
+  fall back to checkpoint `260`.
+
+ETA:
+
+- Remaining active training time from iteration `281` is about `7h12m`.
+- If the resume job starts promptly after the first allocation exits, final
+  training should land around `2026-05-24 12:45-13:15 CEST`.
+- Final conversion/eval should add roughly another hour, so benchmark-ready
+  artifacts are expected around `2026-05-24 14:00 CEST`, plus any queue delay.
+
+## Checkpoint 260 full downstream eval landed
+
+Timestamp: `2026-05-24T03:25:00Z`
+
+Completed job:
+
+- Full downstream eval job `2362713` completed successfully.
+  - Elapsed: `00:44:57`
+  - Exit code: `0:0`
+  - Results:
+    `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000260_full/results_2026-05-24T05-20-34.834207.json`
+
+Local compact evidence copied:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000260_results.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000260_run_metadata.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000260_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000260_new_token_diagnostics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000260_digest.md`
+
+Headline full-eval deltas vs checkpoint `130`:
+
+- General capabilities are mixed to slightly down:
+  - `arc_challenge` acc_norm `0.5367` (`-0.0171`)
+  - `arc_easy` acc_norm `0.7824` (`-0.0177`)
+  - `mmlu` acc `0.5385` (`-0.0100`)
+  - `global_mmlu` acc `0.4467` (`-0.0237`)
+  - `hellaswag` acc_norm `0.7617` (`+0.0019`)
+  - `winogrande` acc `0.6882` (`+0.0032`)
+- Greek movement is mixed but still mostly useful:
+  - `arc_challenge_mt_el` acc_norm `0.3891` (`+0.0375`)
+  - `include_base_44_greek_few_shot_en` acc `0.4112` (`+0.0254`)
+  - `global_piqa_completions_ell_grek` acc_norm `0.5600` (`+0.0300`)
+  - `xnli_el` acc `0.3896` (`+0.0104`)
+  - `belebele_ell_Grek` acc_norm `0.5022` (`+0.0022`)
+  - `global_mmlu_full_el` acc `0.3774` (`-0.0074`)
+  - `xquad_el` exact match `0.1571` (`-0.0210`), F1 `0.3711`
+    (`-0.0003`)
+
+Intrinsic/new-token trend vs checkpoint `130`:
+
+- Heldout BPC improved `0.6531 -> 0.5670`.
+- Heldout NLL/token improved `3.0692 -> 2.6647`.
+- New-target top-1 improved `0.2749 -> 0.3517`.
+- New-target top-10 improved `0.5040 -> 0.5849`.
+- New-vocab probability mass stayed stable: `0.3400 -> 0.3411`.
+- New-row norm ratios and sampled effective rank stayed stable.
+
+Interpretation:
+
+- Checkpoint `260` keeps the strongest signal on intrinsic TD recovery and
+  new-token prediction.
+- Downstream Greek is not uniformly monotonic, but several target Greek metrics
+  improve from checkpoint `130` and most are above checkpoint `65`.
+- The main watch item for final selection is whether the final checkpoint
+  restores or further erodes the small general-benchmark dips seen at `260`.
+
+## Walltime handoff preserved progress at checkpoint 320
+
+Timestamp: `2026-05-24T04:49:24Z`
+
+Main job `2357704` reached the Slurm signal handoff cleanly:
+
+- Last pre-handoff training line: iteration `320/476`, consumed tokens
+  `1.342B`, timestamp `2026-05-24 06:43:29`, loss `2.605217`, throughput
+  `7891.8` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Slurm/Megatron signal path fired after iteration `320`:
+  - `saving checkpoint at iteration     320`
+  - `scheduled an async checkpoint save at iteration     320`
+  - `exiting program after receiving SIGUSR2`
+  - `successfully saved checkpoint from iteration     320`
+- Checkpoint tracker advanced from `260` to `320`.
+- Checkpoint directory now exists:
+  `/capstor/scratch/cscs/fffoivos/runs/bakeoff/td_full25_layer11_2b_20260523T165038Z/checkpoints/iter_0000320`
+- Checkpoint `iter_0000320` contains `common.pt`, `.metadata`,
+  `metadata.json`, and all eight `*.distcp` shards.
+
+Resume job `2357705` started automatically and loaded the handoff checkpoint:
+
+- Resume job state at verification: `RUNNING`.
+- Resume stdout:
+  - `loading distributed checkpoint ... at iteration 320`
+  - `successfully loaded checkpoint ... at iteration 320`
+  - `before the start of training step`
+- Resume stderr contains only expected warnings/deprecations and the xIELU
+  optimizer audit:
+  - rank 0/1/2/3 all report `model_alpha_params=64`,
+    `dp_optimizer_alpha_params=64`, `missing=0`.
+- First resumed training line landed: iteration `321/476`, consumed tokens
+  `1.346B`, timestamp `2026-05-24 06:48:21`, loss `2.629754`, skipped
+  iterations `0`, NaN iterations `0`.
+
+Interpretation:
+
+- The configured `#SBATCH --signal=SIGUSR2@600` plus
+  `--exit-signal-handler` path is proven live, not just code-inspected.
+- The handoff preserved progress beyond checkpoint `260`; the resumed run is
+  continuing from checkpoint `320`.
+- Next planned evaluation checkpoint remains `390`; checkpoint `320` is a
+  resume-safety checkpoint, not a scheduled eval point.
+
+## Resume checkpoint 325 and backup watcher handoff
+
+Timestamp: `2026-05-24T05:05:02Z`
+
+Resume/training state:
+
+- Resume job `2357705` is still running on `normal`.
+- Latest parsed training line at poll: iteration `328/476`, consumed tokens
+  `1.376B`, timestamp `2026-05-24 07:04:15`, loss `2.643836`, throughput
+  `7892.9` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Regular checkpoint `325` saved successfully after the resume.
+- Checkpoint tracker advanced to `325`.
+- Checkpoint `iter_0000325` contains `common.pt`, `.metadata`,
+  `metadata.json`, and all eight `*.distcp` shards.
+
+Watcher handoff:
+
+- First durable watcher `2357765` reached its xfer walltime and ended with
+  state `TIMEOUT`, elapsed `11:50:26`.
+- Backup watcher `2357766` started automatically via
+  `afterany:2357765` and is running on `xfer`.
+- Backup watcher stdout confirms it restarted the watch specs:
+  `65:greek_only:0 130:full:1 260:full:1 390:full:1 455:full:1 476:full:1`.
+- The checkpoint-390 watcher log now contains the backup start marker:
+  `watch start RUN_TAG=td_full25_layer11_2b_20260523T165038Z ITER=390 TASK_GROUP=full`.
+
+Interpretation:
+
+- Training remains healthy after the resume and has made a normal scheduled
+  checkpoint at `325`.
+- Eval automation remains covered through the backup xfer watcher.
+- Next decision-useful checkpoint remains `390`, expected to trigger the full
+  eval plus intrinsic/new-token diagnostics chain.
+
+## Checkpoint 390 reached and eval chain submitted
+
+Timestamp: `2026-05-24T07:27:54Z`
+
+Training/checkpoint state:
+
+- Resume job `2357705` is still running on `normal`.
+- Latest parsed training line at checkpoint: iteration `390/476`, consumed
+  tokens `1.636B`, timestamp `2026-05-24 09:21:40 CEST`, loss `2.556699`,
+  throughput `7881.9` tokens/sec/GPU, skipped iterations `0`, NaN iterations
+  `0`.
+- Checkpoint `iter_0000390` exists and is complete: `common.pt`,
+  `.metadata`, `metadata.json`, and all eight `*.distcp` shards are present.
+- Checkpoint tracker advanced to `390`.
+
+Eval automation:
+
+- Backup watcher `2357766` detected the complete checkpoint and submitted the
+  full checkpoint-390 eval chain.
+- Submitted jobs:
+  - conversion: `2364958` (`tohf_td_l11_390`, `debug`, running at check)
+  - full lm-eval: `2364959` (`eval_td_l11_390_full`, dependency pending)
+  - intrinsic BPC: `2364960` (`bpc_td_l11_390`, dependency pending)
+  - new-token diagnostics: `2364961` (`diag_td_l11_390`, dependency pending)
+- Eval output root:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000390_full`
+
+Interpretation:
+
+- The resumed 2B TD run has reached the next planned decision checkpoint.
+- Eval automation is live after the xfer watcher handoff.
+- At the observed speed, training itself is still on track to finish near
+  `2026-05-24 12:32 CEST`; checkpoint-390 results should land earlier, after
+  conversion plus the full eval queue clears.
+
+## Checkpoint 390 eval completed and copied locally
+
+Timestamp: `2026-05-24T08:13:32Z`
+
+Eval jobs all completed successfully:
+
+- conversion `2364958`: `COMPLETED`, `0:0`, elapsed `00:01:09`
+- full lm-eval `2364959`: `COMPLETED`, `0:0`, elapsed `00:44:40`
+- tokenizer-fair BPC `2364960`: `COMPLETED`, `0:0`, elapsed `00:01:52`
+- new-token diagnostics `2364961`: `COMPLETED`, `0:0`, elapsed `00:02:03`
+
+Compact artifacts copied locally:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000390_results.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000390_run_metadata.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000390_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000390_new_token_diagnostics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000390_digest.md`
+
+Checkpoint-390 headline:
+
+- Intrinsic/new-token recovery continued to improve:
+  - BPC `0.5670 -> 0.5399` vs checkpoint `260`
+  - new-target top-1 `0.3517 -> 0.3787`
+  - greedy new-token utilization `0.196 -> 0.260`
+- Greek classification/reading signals were split:
+  - improved vs `260`: `arc_challenge_mt_el` `+0.0119`,
+    `belebele_ell_Grek` `+0.0311`, `global_mmlu_full_el` `+0.0073`
+  - weaker vs `260`: `xnli_el` `-0.0116`,
+    `include_base_44_greek_few_shot_en` `-0.0127`,
+    `global_piqa_completions_ell_grek` `-0.0100`
+  - largest regression: `xquad_el` exact match `-0.0437`, F1 `-0.0509`
+- General-preservation signals remain mixed:
+  - MMLU/global-MMLU slightly recovered vs `260`
+  - HellaSwag remains slightly above checkpoint `130`
+  - ARC-Challenge, XNLI, and XCOPA are below checkpoint `260`
+
+Training state at the same poll:
+
+- Resume job `2357705` remained `RUNNING`.
+- Latest parsed line: iteration `413/476`, consumed tokens `1.732B`,
+  timestamp `2026-05-24 10:12:42 CEST`, loss `2.544199`, throughput
+  `7866.3` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Training ETA from that line: `2:19:57`, putting finish near
+  `2026-05-24 12:32-12:35 CEST`.
+
+Interpretation:
+
+- The checkpoint-390 evidence strengthens the case that TD is learning the new
+  rows and Greek tokenization target.
+- It does not yet settle the production decision because XQuAD dropped sharply
+  and general preservation remains mixed. Final checkpoint `476`, plus planned
+  checkpoint `455`, should decide whether this is transient or a real
+  over-training/preservation issue.
+
+## Checkpoint 455 reached and eval chain submitted
+
+Timestamp: `2026-05-24T09:54:09Z`
+
+Training/checkpoint state:
+
+- Resume job `2357705` remains `RUNNING`.
+- Checkpoint training line: iteration `455/476`, consumed tokens `1.908B`,
+  timestamp `2026-05-24 11:46:02 CEST`, loss `2.518627`, throughput
+  `7869.0` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint `iter_0000455` saved successfully and tracker advanced to `455`.
+- Training continued past the checkpoint; latest poll after save was
+  iteration `457/476`, consumed tokens `1.917B`, timestamp
+  `2026-05-24 11:50:29 CEST`, loss `2.532910`, skipped iterations `0`,
+  NaN iterations `0`.
+
+Eval automation:
+
+- Backup watcher `2357766` detected the complete checkpoint and submitted the
+  checkpoint-455 full eval chain.
+- Submitted jobs:
+  - conversion: `2366619` (`tohf_td_l11_455`), completed `0:0`,
+    elapsed `00:01:12`
+  - full lm-eval: `2366621` (`eval_td_l11_455_full`), running at last check
+  - intrinsic BPC: `2366622` (`bpc_td_l11_455`), completed `0:0`,
+    elapsed `00:01:40`
+  - new-token diagnostics: `2366623` (`diag_td_l11_455`), completed `0:0`,
+    elapsed `00:02:21`
+
+Quick intrinsic/diagnostic artifacts copied locally:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_new_token_diagnostics.json`
+
+Quick checkpoint-455 intrinsic comparison vs checkpoint `390`:
+
+- BPC: `0.539879 -> 0.532765`
+- NLL/token: `2.537096 -> 2.503662`
+- new-target top-1: `0.378668 -> 0.384541`
+- new-target top-10: `0.612043 -> 0.617435`
+- new-vocab probability mass: `0.344027 -> 0.341786`
+- greedy new-token utilization: `0.260 -> 0.200`
+
+Interpretation:
+
+- Intrinsic likelihood and new-target ranking continued improving at
+  checkpoint `455`.
+- Greedy new-token utilization dropped from checkpoint `390`; this is only a
+  tiny prompt-sample diagnostic, so the full downstream eval remains the
+  decision-useful signal.
+- Next actions are to wait for full eval `2366621`, copy/digest the complete
+  checkpoint-455 evidence, then continue to final checkpoint `476`.
+
+## Checkpoint 455 full eval completed and copied locally
+
+Timestamp: `2026-05-24T10:38:37Z`
+
+Full eval `2366621` completed successfully:
+
+- state `COMPLETED`, exit `0:0`, elapsed `00:46:46`
+- result JSON:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000455_full/results_2026-05-24T12-37-56.749011.json`
+
+Compact artifacts copied locally:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_results.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_run_metadata.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_new_token_diagnostics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000455_digest.md`
+
+Checkpoint-455 full-eval headline:
+
+- Intrinsic recovery continued vs checkpoint `390`:
+  - BPC `0.5399 -> 0.5328`
+  - new-target top-1 `0.3787 -> 0.3845`
+  - new-target top-10 `0.6120 -> 0.6174`
+- Greek downstream vs checkpoint `390`:
+  - recovered: `xnli_el` `+0.0060`, `xquad_el` F1 `+0.0067`,
+    `global_mmlu_full_el` `+0.0006`,
+    `include_base_44_greek_few_shot_en` `+0.0036`
+  - moved down: `arc_challenge_mt_el` `-0.0094`,
+    `belebele_ell_Grek` `-0.0122`,
+    `global_piqa_completions_ell_grek` `-0.0100`
+- Same-checkpoint bakeoff comparison:
+  - TD beats vanilla on Belebele, XQuAD F1, HellaSwag, and MMLU.
+  - Vanilla still beats TD on Greek ARC, Greek MMLU, and the base-44 Greek
+    aggregate.
+
+Training/final checkpoint state at the same close-watch poll:
+
+- Final training reached checkpoint `476/476` at `2026-05-24 12:32:40 CEST`.
+- Checkpoint `iter_0000476` saved successfully and tracker advanced to `476`.
+- Final eval chain submitted:
+  - conversion `2367150`
+  - full lm-eval `2367151`
+  - intrinsic BPC `2367152`
+  - new-token diagnostics `2367153`
+
+Interpretation:
+
+- Checkpoint `455` keeps TD ahead of ReTok/Centroid and confirms the new-token
+  learning path is alive.
+- It does not clearly beat vanilla on the aggregate Greek criteria we care
+  about. Final checkpoint `476` is now the decisive comparison point.
+
+## Final checkpoint 476 reached and quick checks passed
+
+Timestamp: `2026-05-24T10:44:52Z`
+
+Training completed the planned 2B TD run:
+
+- Final training line: iteration `476/476`, consumed tokens `1.996B`,
+  timestamp `2026-05-24 12:32:40 CEST`, loss `2.496399`, throughput
+  `7874.3` tokens/sec/GPU, skipped iterations `0`, NaN iterations `0`.
+- Checkpoint `iter_0000476` saved successfully.
+- Checkpoint tracker advanced to `476`.
+
+Final eval chain submitted and quick side checks completed:
+
+- conversion `2367150`: `COMPLETED`, `0:0`, elapsed `00:01:09`
+- full lm-eval `2367151`: `RUNNING` at last check
+- intrinsic BPC `2367152`: `COMPLETED`, `0:0`, elapsed `00:01:44`
+- new-token diagnostics `2367153`: `COMPLETED`, `0:0`, elapsed `00:02:00`
+
+Quick final artifacts copied locally:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_new_token_diagnostics.json`
+
+Quick intrinsic/diagnostic comparison vs checkpoint `455`:
+
+- BPC: `0.532765 -> 0.531084`
+- NLL/token: `2.503662 -> 2.495764`
+- new-target top-1: `0.384541 -> 0.386419`
+- new-target top-10: `0.617435 -> 0.619114`
+- new-vocab probability mass: `0.341786 -> 0.342453`
+- greedy new-token utilization: `0.200 -> 0.208`
+
+Interpretation:
+
+- The 2B TD training run itself is complete and clean.
+- Intrinsic/new-token learning continued improving through the final
+  checkpoint.
+- The production decision still waits on full final lm-eval `2367151`, because
+  checkpoint `455` remained mixed against vanilla on aggregate Greek tasks.
+
+## Final checkpoint 476 full eval completed
+
+Timestamp: `2026-05-24T11:30:00Z`
+
+Final full lm-eval completed cleanly:
+
+- full lm-eval `2367151`: `COMPLETED`, `0:0`, elapsed `00:45:27`
+- generation phase completed at `2026-05-24 13:20:27 CEST`
+- aggregated results and per-task samples were written under:
+  `/capstor/scratch/cscs/fffoivos/runs/eval/td_full25_layer11_2b_20260523T165038Z/iter_0000476_full`
+
+Compact local artifacts copied:
+
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_results.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_run_metadata.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_tokenizer_fair_metrics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_new_token_diagnostics.json`
+- `eval/live_summaries/td_full25_layer11_2b_20260523T165038Z_iter0000476_digest.md`
+
+Final same-checkpoint TD-vs-vanilla highlights:
+
+- TD wins: `belebele_ell_Grek` `+0.0144`, `xquad_el` F1 `+0.0161`,
+  `mmlu` `+0.0161`, `hellaswag` `+0.0011`, `xnli` `+0.0024`, `xcopa`
+  `+0.0045`; `arc_challenge` is tied.
+- Vanilla wins: `arc_challenge_mt_el` `+0.0171`, `global_mmlu_full_el`
+  `+0.0355`, `include_base_44_greek_few_shot_en` `+0.0145`,
+  `global_piqa_completions_ell_grek` `+0.0500`, and `global_mmlu`
+  `+0.0075`.
+
+Final intrinsic/tokenizer-local highlights:
+
+- TD beats ReTok on heldout BPC: `0.5311` vs `0.5739`, and strongly beats
+  Centroid: `0.8994`.
+- TD beats ReTok on new-target top-1/top-10: `0.3864`/`0.6191` vs
+  `0.3497`/`0.5772`.
+- ReTok still has higher five-prompt greedy new-token utilization: `0.3580`
+  vs TD `0.2080`; this is a weak diagnostic, but worth keeping visible.
+- TD/ReTok/Centroid preserve the Greek compression gain:
+  `3.973` chars/token and `1.735` tokens/word, versus vanilla at `2.557`
+  chars/token and `2.693` tokens/word.
+
+Interpretation:
+
+- TD is the strongest extended-tokenizer path and is no longer a failed
+  initialization: it beats ReTok/Centroid broadly and recovers the new-token
+  objective over the 2B challenger.
+- It still does not clearly beat the vanilla production path on the aggregate
+  Greek/downstream preservation criteria. The safest default for the 15-20B CPT
+  run remains vanilla unless compression/new-token behavior is promoted to a
+  primary objective.

provenance/token-distillation/TOKEN_DISTILLATION_PLAN.md

@@ -0,0 +1,773 @@
+# Token Distillation Parallel Plan
+
+Status: draft execution plan, 2026-05-22
+
+Purpose: prepare a bounded Token Distillation challenger that can start quickly if
+the current Vanilla / ReTok / Centroid bakeoff leaves ReTok promising but
+under-initialized. This is a companion plan, not a change to the running
+three-arm bakeoff.
+
+Primary external references:
+
+- Paper: https://arxiv.org/abs/2505.20133
+- Official implementation: https://github.com/konstantinjdobler/token-distillation
+
+## 1. Decision Summary
+
+Token Distillation is feasible for Apertus without inventing a new method. The
+earlier "bracketed" status was too conservative if it was based on tied-vs-untied
+embedding concerns. The official implementation learns input embeddings by
+hidden-state distillation and has explicit untied-output handling through a
+next-token CE path.
+
+What is still real is integration risk:
+
+- Apertus uses untied `embed_tokens` and `lm_head`, so `E` and `U` must be
+  checked independently.
+- Our tokenizer is an extended BPE tokenizer with fixed production IDs, not a
+  base tokenizer plus `add_tokens(...)`.
+- The distillation run must preserve xIELU and QK-Norm values and must survive
+  the same HF -> Megatron -> HF roundtrip gate as the other arms.
+- Layer choice is not a free constant: the paper defaults to the final hidden
+  layer, while the package README says earlier target layers around the
+  one-third mark can work better. We will run a small layer pilot that tests
+  both: final layer plus the package-suggested one-third layer. If the
+  logit-lens/tuned-lens probe in Section 6.1 suggests a different layer, test
+  that as well.
+
+## 2. When To Start
+
+Do not interrupt the running three-arm bakeoff. Token Distillation can begin
+once all of these are true:
+
+1. The current bakeoff has reached the final scheduled checkpoint, or the
+   final checkpoint is safely saved and the remaining work is only evaluation.
+2. ReTok is not already decisively beaten by Vanilla on both Greek BPC and the
+   new-token diagnostics.
+3. One spare GH200 node can be used without delaying final bakeoff evals.
+4. The implementation runs first on a small token subset and passes all
+   preservation gates in Section 8.
+
+Recommended trigger:
+
+- Run the Token Distillation pilot if ReTok has healthy new-token behavior
+  but still trails Vanilla on Greek BPC or downstream Greek tasks.
+- Do not run it if Vanilla is clearly best and ReTok has weak new-token use;
+  in that case the tokenizer extension itself is probably not helping enough.
+
+## 3. Experiment Shape
+
+Name: `retok_td`
+
+The experiment is not "TD from scratch." It is ReTok plus a TD refinement:
+
+- Logical teacher: the original-tokenization path through the same frozen
+  Apertus weights. In implementation this does not need to be a second 8B model
+  in memory: the extended student can run the unmerged/base-tokenized sequence
+  under `torch.no_grad()` because all original rows and transformer weights are
+  frozen and identical to the base model.
+- Student: ReTok-initialized HF checkpoint with the exact modern-only extended
+  tokenizer used by the current bakeoff.
+- Input objective: hidden-state MSE between teacher runs using base-tokenized
+  spans and student runs using the corresponding single new token.
+- Output objective: next-token CE on the merged sequence, with gradients allowed
+  to update only the new `lm_head` rows.
+- Trainable parameters: new input rows and new output rows only.
+- Frozen parameters: all transformer blocks, all original embedding/output
+  rows, xIELU scalars, QK-Norm weights, RMSNorm weights, and all other base
+  model parameters.
+
+This isolates the question: "Does TD improve ReTok's placement of the new rows?"
+
+## 4. Model Format Plan
+
+Token Distillation should run in HF format, not Megatron format.
+
+Inputs:
+
+- Teacher/reference HF model: the unmodified Apertus HF base, used for validation
+  and optional row-diff checks. Do not require it to stay resident during TD
+  training unless an implementation choice truly needs it.
+- Teacher tokenizer: base 131,072-token Apertus tokenizer.
+- Physical training model: ReTok HF init checkpoint for the modern-only
+  148,480-token tokenizer. This model serves both the merged/student path and
+  the unmerged/logical-teacher path.
+- Student tokenizer: exact shipped extended tokenizer from
+  `03_3_cscs_experiments_kickoff/ship/apertus_greek_modern_only_148480/`.
+
+Outputs:
+
+- HF checkpoint for `retok_td`.
+- Exact student tokenizer copied through unchanged.
+- A manifest with model paths, tokenizer hash, base vocab size, extended vocab
+  size, trainable row ranges, layer choice, data slice, and source commit of
+  the Token Distillation code.
+- Megatron TP=2 `torch_dist` checkpoint produced with the same R17-preserving
+  conversion path used by the current bakeoff.
+
+Sensitive point: do not let the reference implementation call
+`target_tokenizer.add_tokens(...)` for production. That would create added-token
+behavior and may change IDs/segmentation relative to our BPE extension. Patch or
+wrap the implementation so the student tokenizer is loaded directly and treated
+as authoritative.
+
+Second sensitive point: do not load two 8B models just because the method is
+described as teacher/student. The official training loop uses one extended model
+for both the merged and unmerged forwards; preserve that memory-efficient shape
+unless a later validation test proves it insufficient.
+
+## 5. Token Mapping And Data
+
+For each new token ID in the extended tokenizer:
+
+1. Decode the token string using the same ByteLevel-aware readable-token logic
+   used in the tokenizer audits.
+2. Tokenize the same surface string with the base tokenizer to get the teacher
+   phrase IDs.
+3. Tokenize it with the extended tokenizer and verify it maps to exactly the
+   intended new token ID in the relevant context.
+4. Mine snippets from the CPT Greek corpus where that surface string occurs.
+
+Use real corpus snippets, not generated snippets, for the main run. Generated
+snippets are acceptable only as a fallback for very low-coverage tokens in a
+separate flagged bucket.
+
+Initial data budgets:
+
+- Smoke: 256 to 512 high-frequency new tokens, 20 to 50 snippets per token.
+- Layer pilot: 1,024 to 2,048 tokens, 50 snippets per token, comparing
+  last-layer TD against the package-suggested one-third-depth layer and, if
+  available, the Section 6.1 probe-suggested layer.
+- Full modern challenger: all 17,408 modern new tokens at the paper-fast
+  `25`-snippet setting first. The `100`-snippet setting is supported by the
+  coverage result, but is a follow-up only if the fast full-token checkpoint
+  clears gates while leaving quality ambiguous.
+- Production polytonic extension: only after the modern path passes; then repeat
+  for the additional +5,120 polytonic tokens.
+
+Coverage policy:
+
+- Do not silently skip tokens. Emit `coverage.json` with requested snippets,
+  found snippets, used snippets, and fallback action per token.
+- Split results into enough-coverage and low-coverage buckets during analysis.
+- If more than 10 percent of new tokens have fewer than 20 real snippets, stop
+  before full TD and inspect the token inventory/corpus mismatch.
+
+## 5.1 Firing Prepass On The Exact 2B Dataset
+
+Before any GPU TD run, count actual new-token firings and usable snippets on the
+exact normalized text/parquet slice that produced the 2B-token bakeoff dataset.
+Do this on the pre-packed source text, not only on Megatron `.bin`, because TD
+needs text spans and 50-token contexts.
+
+This is a CPU/I/O job and should run as a sidecar only after the current bakeoff
+final checkpoints are safely saved, or while final downstream evals are running.
+Do not let it compete with training checkpoint saves or packed eval cache reads.
+
+Output artifact:
+
+- `td_coverage_prepass.jsonl`: one row per new token.
+- `td_coverage_summary.json`: aggregate thresholds and recommended action.
+- `td_snippet_index/`: sampled snippet references or extracted tokenized snippets
+  keyed by `new_token_id`.
+
+Required row fields:
+
+- `new_token_id`
+- `token_string`
+- `base_subtoken_ids`
+- `base_subtoken_len`
+- `extended_firings`
+- `raw_surface_occurrences` where cheap to compute
+- `usable_snippets_25`
+- `usable_snippets_100`
+- `docs_with_firing`
+- `example_snippet_refs`
+- `status`: `enough_100`, `enough_25`, `low_20_24`, `low_lt20`,
+  `zero`, or `mismatch`
+- `recommended_action`: `td_100`, `td_25`, `keep_retok`,
+  `generated_fallback_candidate`, or `inspect`
+
+Counting rule:
+
+- A firing means the extended tokenizer actually emits that new token ID.
+- Merge ancestry does not count.
+- A surface substring does not count unless it corresponds to the emitted
+  new-token span under the extended tokenizer.
+- For each firing span, tokenize the same surface span with the base tokenizer
+  to build the logical-teacher sequence.
+
+Decision gate:
+
+- If at least 90 percent of new tokens have `usable_snippets_100 >= 100`, mark
+  the corpus as high-coverage, but still run the first full-token challenger at
+  the paper-fast 25-snippet setting unless a prior full-token fast run has
+  already passed.
+- If at least 90 percent have `usable_snippets_25 >= 25`, run TD at the paper's
+  fast 25-snippet setting and keep a flagged tail strategy.
+- If more than 10 percent have `usable_snippets_25 < 25`, do not launch full TD.
+  Inspect whether the problem is dataset coverage, tokenizer/string decoding, or
+  low-value tail tokens.
+- Tokens with `usable_snippets < 20` keep ReTok by default. Generated snippets
+  are allowed only as a separately reported fallback bucket.
+- Tokens with `status = mismatch` block the TD run until the tokenizer/span
+  mapping is fixed.
+
+## 6. Layer Choice
+
+**Default candidate for Apertus 8B: last layer (`target_layer = -1`). Required comparison candidate: package-suggested one-third-depth layer. No broad sweep.**
+
+The paper argues for last layer as the conservative, portable default. Direct quotes:
+
+> *"In practice, we simply use the last layer's hidden state but analyze this choice in Section 5.3."* — paper §3.2, p.4.
+
+> *"For our main experiments, we choose to keep the last layer because this choice does not necessitate a model-specific sweep over target layers. Also, the last layer is a principled choice, as it guarantees that no subtoken interactions that are only modeled in later layers are excluded from the objective."* — paper §5.3 (per a second-pass review of the paper's layer-choice analysis).
+
+The argument for the last layer is **conservative-and-correct**: any way the original subtoken sequence influenced downstream computation must, by construction, have shown up by the final hidden state. Matching there upper-bounds the objective. But the package README is explicit that earlier target layers around the one-third mark can yield better results in many cases. That means we should not pick last-layer only without checking the package-suggested alternative.
+
+Implication for Apertus 8B specifically: run a **small** layer pilot, not a sweep.
+
+- Candidate A: `target_layer = -1`, the paper-default last layer.
+- Candidate B: `target_layer = ceil(num_hidden_layers / 3)` in HF hidden-state tuple indexing after confirming Section 16 Q11. For Apertus-8B's 32 transformer layers, expected candidate is `target_layer = 11` (roughly after the first third of blocks).
+- Candidate C: if the Section 6.1 logit-lens/tuned-lens probe suggests a materially different detokenization layer `L*`, test that layer too. If `L*` is within +/-1 of Candidate B, do not add a third candidate.
+- Implementation note from the first live smoke: the vendored package does **not** ship a separate layer-suggestion tool. Its actionable upstream guidance is the README note that one-third-depth target layers can work better than the last layer. Therefore the first layer pilot should compare Candidate A and Candidate B only, unless we have a well-defined Apertus-specific probe result.
+
+Selection rule:
+
+- Choose the candidate with better heldout Greek BPC/NLL and D1/D2/D4/D5 on the layer-pilot subset.
+- If differences are within noise, choose last layer for portability and to avoid model-specific overfitting.
+- Do not test more than three candidates unless the first TD run fails in a way that is clearly layer-specific.
+
+For polytonic Phase 2 specifically (the +5,120 polytonic tokens, deferred per §5): **also default to last layer**. The detokenization machinery for unfamiliar diacritic-heavy sequences plausibly uses more late-layer compute than for typical modern words — the model has to reassemble diacritic + base-char interactions it saw less cleanly during pretraining. The conservative default earns its keep here.
+
+### 6.1 Layer-suggestion probe to find the "detokenization layer" L*
+
+Before the layer pilot, run a cheap logit-lens probe to get our best Apertus-specific guess for where the residual stream finishes resolving multi-token Greek words into a single contextual representation:
+
+1. Take a held-out set of multi-token Greek words from our corpus (~200-500 words that the base tokenizer fragments into 2-5 subpieces).
+2. Run them through the **original Apertus base** (no TD), recording the residual stream at every layer at the position of the LAST subtoken.
+3. Project each layer's residual through Apertus's `lm_head` (logit lens) OR a learned per-layer linear probe (tuned lens — Belrose et al. 2023, *Eliciting Latent Predictions from Transformers with the Tuned Lens*, arXiv:2303.08112).
+4. Find the layer index `L*` where the top-1 projection first reliably matches the whole-word identity (i.e., where the model has "decided" what whole word it's processing).
+
+For Llama-class 32-layer 8B models, `L*` is reportedly in the first third of layers (per a Kaplan et al. 2025 "Tokens to Words" line of work — **citation needs verification before being load-bearing**). For Apertus, plausibly similar (`L* ≈ 8-12` of 32), but Apertus's xIELU + QK-Norm could shift this; the only way to know is to run the probe.
+
+**Cost:** one forward pass per probe sample × 32 layers of projection ≈ trivial on one GH200 (< 30 min for 500 words).
+
+**What it gives us:** an *informative diagnostic* for the paper-quality writeup ("here's where Apertus does detokenization"), and a sanity check that last-layer TD isn't wasted compute (if `L* = 5` and we train at layer 32, we're propagating identical signal through 27 layers per backward pass). If we find `L* = 25+`, that's also informative — Apertus's late layers may be more LM-head-staging than Llama's, in which case last-layer is doubly the right choice.
+
+**Action:** run the cheap logit-lens version after the first successful TD smoke
+and before the layer pilot. If it suggests a clear `L*` that differs from the
+one-third-depth layer, include `L*` as Candidate C in the layer pilot. The
+heavier tuned-lens version remains optional and should not block the modern TD
+path. Tracked as task TD7 in §9.
+
+## 7. Output Embeddings
+
+Apertus has untied embeddings, so `lm_head` rows must be handled deliberately.
+
+Primary output strategy:
+
+- Start from the ReTok `lm_head` rows already present in the student checkpoint.
+- During TD, train only the new output rows with next-token CE.
+- Keep original output rows frozen and verify they are unchanged exactly.
+- Keep the input hidden-state MSE and output CE accounting separate in logs.
+
+Fallbacks:
+
+- If CE makes output norms unstable, train only input rows and keep ReTok output
+  rows fixed.
+- If ReTok output rows look bad before TD, use subtoken-mean output initialization
+  from the base output matrix, then CE-train only new output rows.
+
+Avoid using a tied-embedding shortcut. It does not apply to Apertus.
+
+## 8. Verification Gates
+
+The TD run is not usable until all of these pass.
+
+Tokenizer gates:
+
+- Extended tokenizer files are byte-identical before and after TD.
+- Base special token IDs and extended special token IDs match the current
+  bakeoff tokenizer manifests.
+- New token IDs equal the planned contiguous range.
+- A fixed sample of Greek strings tokenizes exactly as in the ReTok arm.
+
+Model gates:
+
+- Original input rows unchanged exactly.
+- Original output rows unchanged exactly.
+- New input/output rows finite, nonzero, and within documented norm envelopes.
+- xIELU scalar tensors unchanged exactly.
+- QK-Norm tensors unchanged exactly.
+- Logits on prompts that do not use new tokens remain unchanged or within a
+  documented numerical tolerance.
+
+Roundtrip gates:
+
+- Convert HF `retok_td` -> Megatron TP=2 with the R17-preserving path.
+- Convert back or run the existing roundtrip verifier.
+- Require zero or near-zero diff for xIELU, QK-Norm, and unchanged base rows.
+- Run a short Megatron load/train smoke before queueing any CPT-scale arm.
+
+Quality gates:
+
+- Pre-CPT BPC should improve over ReTok on the heldout Greek slice.
+- D1/D2/D4/D5 should improve or remain healthy versus ReTok.
+- No obvious English retention regression on the small prompt/eval sanity set.
+- If pre-CPT gates are neutral, do not spend a full 2B-token arm on TD.
+
+## 9. Implementation Tasks
+
+Task TD0 - vendor or pin reference code:
+
+- Pin the official Token Distillation repo commit in a manifest.
+- Do not rely on a moving GitHub checkout from inside a Slurm job.
+
+Task TD1 - exact-tokenizer adapter:
+
+- Add a wrapper that accepts separate teacher and student tokenizers.
+- Disable `add_tokens(...)` in the production path.
+- Build explicit mapping:
+  `new_token_id -> token_string -> base_phrase_ids -> student_token_id`.
+
+Task TD2 - local corpus snippet source:
+
+- Implement a local JSONL/parquet snippet miner over the existing CPT corpus
+  slice.
+- Cache snippets in a reusable artifact keyed by tokenizer hash and corpus hash.
+- First mode is the Section 5.1 firing prepass: count actual extended-token
+  firings and usable snippets on the exact 2B bakeoff text slice, then produce
+  `td_coverage_prepass.jsonl`, `td_coverage_summary.json`, and
+  `td_snippet_index/`.
+
+Task TD3 - embedding-only trainer:
+
+- Load the ReTok HF student once and use it for both the merged/student forward
+  and the unmerged/logical-teacher forward.
+- Freeze all non-new rows and all transformer parameters.
+- Backprop hidden-state MSE to new input rows.
+- Backprop CE to new output rows only.
+- Emit loss, norm, gradient, and coverage telemetry.
+
+Task TD4 - pilot runs:
+
+- Run smoke on 256 to 512 high-frequency new tokens.
+- Only choose smoke tokens from `status in {enough_100, enough_25}` in the
+  Section 5.1 prepass.
+- Run the Section 6.1 logit-lens probe to get a candidate detokenization layer
+  `L*`.
+- Run a layer pilot on 1,024 to 2,048 tokens comparing:
+  - `target_layer=-1` (paper default),
+  - `target_layer=ceil(num_hidden_layers / 3)` (package README suggestion;
+    expected Apertus-8B value: `11` after Q11 indexing confirmation),
+  - `target_layer=L*` if the probe suggests a materially different layer.
+- Pick the output strategy before full TD.
+
+Task TD5 - full modern TD:
+
+- Run all 17,408 modern new tokens first at `snippets_per_token=25`.
+- Keep `snippets_per_token=50/100` as a follow-up quality-budget test, not the
+  first full-token cost event.
+- Save HF checkpoint, manifest, coverage, and diagnostics.
+- Run verification gates.
+
+Task TD6 - optional fourth CPT arm:
+
+- Only after TD5 passes, convert to Megatron and run a short CPT smoke.
+- If smoke passes and pre-CPT diagnostics beat ReTok, submit `retok_td` as a
+  separate fourth arm or a shorter challenger run.
+
+## 10. Risks And Mitigations
+
+Tokenizer mismatch:
+
+- Highest-risk integration point. Mitigation: never regenerate the tokenizer;
+  load the exact extended tokenizer and assert ID/sample-tokenization equality.
+
+Layer mis-selection:
+
+- Mitigation: test the paper-default last layer against the package-suggested
+  one-third-depth layer and, if available, the tool/probe-suggested layer.
+  Keep the candidate set tiny to avoid turning TD into a layer-sweep project.
+
+Output-row instability:
+
+- Mitigation: keep ReTok `U` as the starting point, CE-train only new rows, and
+  fall back to fixed ReTok `U` if norms or logits become unstable.
+
+Silent base-model drift:
+
+- Mitigation: exact row-diff checks after every pilot and full run.
+
+xIELU/QK-Norm drift:
+
+- Mitigation: freeze all non-embedding parameters and reuse the R17 roundtrip
+  verifier.
+
+Coverage skew:
+
+- Mitigation: coverage manifest, frequency buckets, and a hard stop if too many
+  tokens lack real snippets.
+
+Resource overuse:
+
+- Mitigation: one-node pilots first; do not queue a full fourth arm before the
+  current bakeoff result is interpretable.
+
+## 11. Expected Cost
+
+Order-of-magnitude expectation:
+
+- smoke: less than 1 node-hour,
+- layer pilot: 1 to 2 node-hours,
+- full modern TD: roughly 5-7 wall hours for the first full-token 25-snippet
+  challenger on one GH200-class process; more if run at 50/100 snippets,
+- optional CPT challenger: same training cost shape as one current bakeoff arm.
+
+The TD preparation is worth doing now because implementation and verification
+can be made ready without spending the CPT-scale cost event.
+
+> Compute reality check (paper, page 4): *"These restrictions ensure that our method is quick to run, initializing 2,500 new tokens on a single GPU in under 10 minutes."* (1× H100 80GB, AdamW, 25 snippets × 50 ctx tokens.) Our live Apertus layer pilot, however, is the better planning anchor: `1,020` trained tokens × `50` snippets at layer 11 took `1,966.9s`. Scaling that to `17,392` trainable tokens at `25` snippets gives roughly **4.7 wall hours** for the fast full-token run; scaling to `100` snippets gives roughly **18-19 wall hours**. Because Clariden normal nodes expose 4 GPUs and have a 12h limit, the first full-token TD challenger should be `25` snippets. A 50/100-snippet run is a separate follow-up, not the first full-token cost event.
+
+---
+
+## 12. Apertus-specific considerations
+
+These are questions about how Apertus's architectural quirks interact with the official TD implementation. Each is tagged with priority and a `Resolution` line that gets filled in once verified.
+
+### A1. Position-encoding asymmetry between teacher and student sequences (RoPE phase shift)
+
+**The mechanism.** When the teacher reads the unmerged sequence `[ctx_L, s1, ..., sk, ctx_R]` and the student reads the merged sequence `[ctx_L, T, ctx_R, PAD × (k-1)]`, the RoPE phase at every position *after* the merged-phrase position differs by `(k-1)` between teacher and student. Apertus uses RoPE with llama3-style scaling factor 8.0, so the phase difference manifests in the q/k rotations the model applies before the dot product.
+
+**Resolution (from official code, train_loop.py:67-89):** the implementation **does** pad the merged sequence with PAD tokens at the end to match the unmerged length. Both teacher and student therefore see the same total sequence length, but the meaningful tokens occupy different positions:
+- Teacher: real tokens at positions `[0, L+k+R-1]`, no padding.
+- Student: real tokens at positions `[0, L+R-1]`, PAD at positions `[L+R, L+k+R-1]`.
+
+The MSE loss is taken over positions selected by `unmerged_to_merged_mask == 1` (teacher) and `merged_seq != pad_id` (student). These two selections produce the same number of positions. The compare position for the assigned-phrase's last subtoken (teacher position `L+k-1`) is matched to the student position `L` (the single new token).
+
+**Implication:** the RoPE phases at the compare positions for `ctx_R` tokens *differ between teacher and student* by `(k-1)`. The implementation accepts this and trusts that the model's downstream layers still produce comparable hidden states. The paper does the same. For modern Greek where `k` is typically 2-5, this is mild; for polytonic where `k` can be larger (more aggressive subword fragmentation), it's more pronounced. **Status: accepted as the paper's design**; we don't need to change it, but it's worth flagging as a known approximation.
+
+**Action:** none for modern bakeoff. For the polytonic Phase 2 sub-plan (§9.5 below), consider whether `k > 8` tokens should be excluded from the TD set.
+
+### A2. Cached teacher hidden states — optional optimization, not first implementation
+
+The official `train_loop.py:282-285` runs the teacher forward fresh on every epoch under `torch.no_grad()`. The teacher is frozen, so its hidden states for each `(snippet, target_position)` pair are **fixed across all epochs**. The plan should add caching:
+
+- After dataloader construction, do one teacher forward pass over the whole dataset, save `og_hiddens[mask==1]` to disk keyed by snippet hash + target layer.
+- During the training loop, skip the teacher forward entirely; read cached hiddens.
+
+**Cache footprint:** ~17,408 tokens × 100 snippets × `~50` positions × 4096 dim × bf16 ≈ **700 GB per target layer**. Too big for one node's RAM and large enough that random per-batch reads could become an I/O problem if the dataloader shuffles globally.
+
+If we want a tighter cache: store only the positions used by the loss (mask=1 positions), which is at most `~50` per snippet. Same order of magnitude.
+
+**Cache key must include:** teacher checkpoint hash, dtype (bf16/fp32), `attn_implementation` (sdpa/flash-attn-2), `target_layer`, RoPE config, NFC version of the snippet text. Invalidation must trigger a fresh cache.
+
+**Speedup:** potentially useful when repeating the same data/layer across several
+LR or batch-size probes. For the first one-epoch run, caching may not beat the
+simplicity of recomputing the teacher forward, especially if cache reads become
+random.
+
+**Action:** do **not** make caching a blocker for TD1-TD5. Implement uncached
+first, then add a cache only if the layer pilot repeats the same target layer or
+the teacher forward is empirically dominating wall time.
+
+### A3. NFC normalization on snippets — must match the corpus we trained on
+
+`apertus_fidelity_checklist.md` V9: the CPT corpus is NFC-normalized in place by `normalize_nfc.sh`. The TD snippet miner must read from the **post-normalize parquet** (`cpt/selected_after_apertus_and_internal_dedup.parquet` after job 2335826), not the pre-normalize version. Any TD that conflates the two will see surface-string mismatches (e.g., NFD-decomposed Greek diacritics will not match new-token strings whose tokenizer was trained on NFC).
+
+**Action:** assert in the snippet miner that the corpus was last modified after the normalize_nfc job's recorded timestamp; refuse to run if not.
+
+### A4. Mixed-precision recipe for TD
+
+The official code uses `torch.autocast("cuda", dtype=torch.bfloat16)` with `mixed_precision=True`. This matches Apertus's bf16-main / fp32-master-grads pretraining recipe. **Recommendation:** keep the default. AdamW moment buffers are fp32 by default in PyTorch; with `weight_decay=0` (paper default) this is identical to Apertus's optimizer state for the trainable rows.
+
+### A5. Apertus-overlap-drop applied to the CPT corpus — soft consideration
+
+The CPT corpus has `apertus_overlap_drop_docs.parquet` rows removed (the docs that overlap with Apertus's pretraining set). The TEACHER (Apertus base) has *seen* those dropped docs and presumably has sharper hidden states on them. Snippets drawn from the post-drop CPT corpus give a slightly noisier teacher.
+
+For the modern-Greek TD, this is a minor effect — the corpus is still ~5B Greek tokens, plenty for snippets. But for the polytonic Phase 2 (where the corpus is thinner anyway), drawing some snippets from the pre-drop set could help. The pre-drop corpus exists at `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/nanochat/data/*.parquet` (before the apertus-overlap drop overlay was applied).
+
+**Action:** for modern TD, ignore. For polytonic TD, consider adding ~10-20% snippets from the pre-drop corpus to the pool.
+
+### A6. Polytonic snippet sourcing — a separate Phase 2 data problem
+
+The plan's §5 mentions polytonic TD as a follow-on. The modern-only CPT corpus has almost no polytonic content (the corpus was Greek-modern-targeted). For the +5,120 polytonic tokens, snippets must come from elsewhere:
+
+- The polytonic-marked subsets of the original GlossAPI corpus (pre-Apertus-overlap-drop).
+- External polytonic sources: ancient/koine Wikipedia, Project Gutenberg Greek, the patristic / classical Greek corpus subsets.
+- The `legacy/corpus_clean_normalization/` polytonic-flagged documents (per `glossapi_corpus_cli/pipeline.py`'s `derive_historical_flag`).
+
+**Action:** Phase 2 TD requires a polytonic snippet pull as a prerequisite. Add as a separate work item; do not block modern TD on it.
+
+---
+
+## 13. Paper and repo specifics — verifications from the research pass
+
+Each row records what the plan was assuming, what the paper / repo actually says, and the resulting action.
+
+### P1. The implementation uses `model.forward()` outputs, not custom attention hooks
+
+**Plan assumption:** "Use `model.forward()` rather than reimplemented attention math" (per `cpt_plan.md v0.7 §13`'s general advice).
+
+**Verified:** `train_loop.py:278` calls `model(merged_seq, output_hidden_states=True)`, `train_loop.py:285` calls `model(unmerged_seq, output_hidden_states=True)` under `torch.no_grad()`. The hidden states are read from `outputs["hidden_states"][target_layer]`. Standard HuggingFace API. **Compatible with Apertus's xIELU + QK-Norm** because both happen inside `model.forward()`.
+
+**Action:** none. The QK-Norm worry from v0.6 §13 is genuinely moot for hidden-state TD.
+
+### P2. Learning rate, optimizer, scheduler, step count
+
+**Plan assumption:** "LR ~1e-3 to 1e-4 AdamW, ~500-2000 steps".
+
+**Verified from paper (p.4) + code (`train_loop.py:250-259`):**
+- Optimizer: AdamW, `weight_decay=0.0`.
+- Scheduler: **constant LR** (code comment: *"paper used linear warmup + decay but constant works fine and might even converge faster"*). The paper itself says *"linear warmup, constant rate"*.
+- Default LR in code: `1e-4`. README: *"learning rates around 1e-4 work well for many models"*.
+- Paper: *"For fair comparison, we sweep for the best learning rate for all methods that require a learning rate."* — they sweep per method.
+- Step count: not directly named; controlled by `epochs * len(dataloader)`. Default `epochs=1`. With 17,408 tokens × 100 snippets / batch_size 16 ≈ 109,000 batches per epoch.
+
+**Action:** start with `lr=1e-4, epochs=1, weight_decay=0, scheduler=constant` (paper + repo defaults). LR sweep is a Phase 2 optimization if results are off; do not sweep on the first run.
+
+### P3. Snippet count + length recommendations
+
+**Plan assumption:** "100 snippets per token, snippet length not specified."
+
+**Verified from paper (p.4) + README:**
+- **Fast setting (used for main results):** `snippets_per_token=25, snippet_len=50`. *"2,500 new tokens on a single GPU in under 10 minutes"*.
+- **Larger compute budget (paper §5.2):** `snippets_per_token=100, snippet_len=50`.
+- README: *"We observe diminishing returns for scaling `snippet_len` beyond 50 and `snippets_per_token` beyond 100."*
+
+**Recommendation after live Apertus pilot:** start the full-token challenger at
+the paper-fast budget (`25, 50`). The coverage result says the corpus can
+support `100` snippets for 99.82 percent of the new tokens, but the live runtime
+estimate puts `100` snippets at roughly 18-19 wall hours on one GPU process.
+Use `50/100` only after the fast full-token checkpoint is evaluated.
+
+**Action:** set `snippets_per_token=25, snippet_len=50` for the first full
+modern run. Keep `snippets_per_token=100` as a documented optional high-budget
+follow-up.
+
+### P4. Snippet sampling and batch composition
+
+**Verified from code (`tokdist.py:191-278`):**
+- **Pattern matching:** uses Aho-Corasick (`ahocorasick.py`) to find all occurrences of new-token surface strings in a tokenized corpus, in a single linear pass.
+- **Snippet size:** `offset_before = snippet_len*2`, `offset_after = snippet_len*2` (so candidates have up to ~4× snippet_len + pattern_len tokens), then truncated to exactly `snippet_len` with the pattern roughly centered.
+- **Filtering:** if fewer than `snippets_per_token` usable snippets are found, the token is **skipped** (printed to stdout, returned in the `skipped` list). The plan's §5 coverage policy already mirrors this.
+- **Shuffling:** `random.shuffle(truncated)` before truncation to `snippets_per_token`, to avoid sort-order bias.
+- **BOS prefix:** `bos_prefix = [bos_token_id]` is prepended to every snippet.
+- **Batches:** dataset is flat (one entry per (snippet, assigned_phrase) pair), dataloader uses `shuffle=True` so batches mix tokens.
+
+**Action:** mirror this snippet-mining logic in our adapter; replace `HFDataSource.load_dataset(...)` with PyArrow streaming over our local CPT corpus parquets (the official code only knows how to read HF datasets).
+
+### P5. Loss normalization
+
+**Verified from code (`train_loop.py:288-300`):**
+- `torch.nn.functional.mse_loss(...)` with default `reduction='mean'` over `(num_selected_positions × hidden_dim)`.
+- Teacher selection: `og_hiddens[unmerged_to_merged_mask == 1]` — selects positions of unmerged tokens NOT in the merged phrase, plus the LAST subtoken position of the merged phrase. Sum equals length of merged seq (before padding).
+- Student selection: `token_distillation_hiddens[merged_seq != pad_id]` — non-pad positions. Same length.
+
+**Implication:** the loss is averaged over **every position that the model produces in the merged sequence**, not only the new-token position. This means the gradient signal includes "your downstream context also has to match the teacher's downstream context after the substitution." Subtle but important: it pushes the new embedding to be compatible with *what the teacher does next*, not just to match the teacher *at the new token position*.
+
+This is consistent with the paper's intuition (§3.1): *"the multiple subtokens t1, ..., tn have on other tokens attending to them in succeeding positions after seeing t*"*.
+
+**Action:** preserve this behaviour in our adapter; do not "fix" it to be new-token-position-only.
+
+### P6. Output embeddings for untied models
+
+**Plan assumption:** train new U rows via NTP on the same data.
+
+**Verified from paper (p.4) + code (`train_loop.py:339-348`):**
+- Paper: *"Since our method backpropagates gradients back from the hidden states, we do not learn output embeddings with our distillation-based objective."*
+- Paper: *"In practice, for learning output embeddings, we can simply add a next-token prediction objective just for the output embeddings at a minimal computational overhead."*
+- Code: separate `torch.autograd.backward([ce_loss], inputs=[lm_head.weight], retain_graph=True)` for output, and `loss.backward(inputs=[input_embeddings.weight])` for input.
+- The original-row gradients are zeroed for both input and output (`train_loop.py:354-359`) before the optimizer step, so base rows never drift.
+
+**Important nuance from paper (p.7):** *"a subtoken attention distillation objective is superior to next-token prediction for learning new token embeddings, the combination via a sum of the two objectives (Token Distillation + NTP) generally yields worse results than Token Distillation alone."* — so combining input MSE + input NTP on the SAME parameter is worse. The split (MSE on E, CE on U) is the right pattern.
+
+**Action:** use `learn_output_with_ce=True` for the untied Apertus path. Train new U rows via the CE-on-merged-logits-of-next-token path. The CE loss is only computed on the merged sequence, not the unmerged.
+
+### P7. The "earlier target layer" claim — verified, but paper still defaults to LAST
+
+**Plan assumption (§6):** *"the authors note that roughly one-third-depth layers often work better."*
+
+**Verified from paper (p.4, §3.2 last paragraph):** *"In practice, we simply use the last layer's hidden state but analyze this choice in Section 5.3."*
+
+**Verified from code (`tokdist.py:67`):** `target_layer: int = -1` default. Comment: *"or e.g. an early layer such as 6 or 7"* — a hint, not a claim of superiority.
+
+**Verified from paper §5.3:** the authors report a slight downward trend as the
+target approaches the last layer, and say earlier layers can improve results and
+speed. They still use the last layer for main experiments because it avoids a
+model-specific sweep and is principled: later subtoken interactions cannot be
+excluded.
+
+**Action:** run the small layer pilot in §6 before full modern TD: final layer
+vs the one-third-depth layer, plus the probe-suggested layer if different.
+Candidate selection uses the cheap gates in §6, not the full downstream suite.
+
+---
+
+## 14. Implementation specifics for our setup
+
+### 14.1 We must NOT use the high-level `TokenDistillation.run(...)` entry point
+
+The official `run()` method calls `target_tokenizer.add_tokens(new_tokens)` (`tokdist.py:97, 523`) which **appends** new tokens to the source tokenizer. This is incompatible with our setup because:
+
+- Our extended tokenizer (`apertus_greek_modern_only_148480/`) was built by **merge-rule extension** (`model.vocab` + `model.merges`), not by `add_tokens(...)`.
+- The new-token IDs in our tokenizer occupy the range `[131072, 148480)` in a specific order determined by our BPE merge order.
+- If `add_tokens(...)` is called on the original base tokenizer with our new-token strings, the resulting IDs will match our planned range *only if* the order matches — which is fragile and unverified.
+
+**Action:** drop into the lower-level `train_embeddings()` function directly, with:
+- Student `model = AutoModelForCausalLM.from_pretrained(retok_init_checkpoint)` — already has the 148,480 vocab.
+- Student tokenizer loaded directly from our shipped extended bundle.
+- Skip `_build_target_tokenizer`; skip `extend_pretrained_with_tokens_and_embeddings`.
+- Construct `phrase_to_new_id` and `assigned_new_phrases` from our **known** new-token IDs.
+
+### 14.2 Snippet miner: PyArrow batch-streaming over local parquet, not HF datasets
+
+The official `build_snippets_for_tokens_from_hf` uses `datasets.load_dataset(streaming=True)` over an HF dataset. Our corpus is local parquet on iopsstor. Replacement plan:
+
+- Read `cpt/selected_after_apertus_and_internal_dedup.parquet` via `pyarrow.parquet.ParquetFile(...).iter_batches(batch_size=N)`.
+- Tokenize each batch with the **base 131,072 tokenizer** (same as paper's "source tokenizer").
+- Run Aho-Corasick (use the official `ahocorasick.py`, no need to reimplement) over the tokenized stream.
+- Reuse the truncation + filtering logic from `build_snippets_for_tokens_from_hf` (offset_before, offset_after, snippet_len, snippets_per_token).
+- Cache the tokenized parquet on iopsstor scratch keyed by `tokenizer.name_or_path + parquet hash`.
+
+The tokenized-corpus cache (from the paper's `_tokenize_dataset_if_needed`) is **a real time-saver**: tokenizing the 47M-row Greek pool once with the base tokenizer takes ~30-60 min on one node. Don't re-tokenize per TD run.
+
+### 14.3 Frozen-parameter table (replaces vague "freeze everything else" in §3)
+
+| Tensor | Trained? | Notes |
+|---|---|---|
+| `model.embed_tokens.weight[131072:148480, :]` | **YES** (MSE-on-hiddens loss) | new input rows |
+| `lm_head.weight[131072:148480, :]` | **YES** (CE loss on merged seq) | new output rows |
+| `model.embed_tokens.weight[:131072, :]` | NO (grad zeroed after backward) | base input rows, verified unchanged at end |
+| `lm_head.weight[:131072, :]` | NO (grad zeroed after backward) | base output rows, verified unchanged at end |
+| `model.layers.*.self_attn.{q,k,v,o}_proj.weight` | NO | attention QKV + output |
+| `model.layers.*.self_attn.{q_norm, k_norm}.weight` | NO | QK-Norm (R17 sensitive — DO NOT TOUCH) |
+| `model.layers.*.mlp.{up_proj, down_proj}.weight` | NO | MLP (xIELU lives between these) |
+| `model.layers.*.mlp.act_fn.{alpha_p, alpha_n, beta, eps}` | NO | xIELU scalars (R17 sensitive — DO NOT TOUCH) |
+| `model.layers.*.input_layernorm.weight` | NO | RMSNorm input |
+| `model.layers.*.post_attention_layernorm.weight` | NO | RMSNorm post-attn |
+| `model.norm.weight` | NO | final RMSNorm |
+
+The official `train_embeddings` already implements this freeze pattern correctly via `param.requires_grad=False` for everything except E/U, and zeros gradients on the original rows after backward. **Verify post-TD that R17-sensitive tensors are unchanged** (use `verify_hf_roundtrip.py`'s R17/xIELU/QK-Norm diff functions).
+
+### 14.4 Padding direction — RIGHT-pad, not left
+
+Code (`train_loop.py:106-140`) right-pads the merged sequence to the unmerged length. The loss explicitly excludes pad positions on both teacher and student sides. This is correct *for this objective* — the trailing PAD positions are not used by the loss, and causal attention from non-pad positions doesn't attend forward to pad.
+
+**Action:** when porting to our adapter, keep right-padding. Don't try to "fix" it to left-padding (which would break the position-mapping logic).
+
+### 14.5 Single-GPU vs DDP
+
+Paper experiments are on 1× H100. Clariden normal nodes expose four GH200 GPUs
+as a node unit; there is no true single-GPU partition. **Recommendation:
+single-process TD for the first full-token run to stay close to the official
+implementation, but pack any genuinely useful independent comparison into the
+same allocation.** For the immediate full-token challenger, that means layer 11
+as the selected candidate and optionally last-layer as the conservative backup,
+both at 25 snippets. Do not invent DDP or row-sharded merge machinery unless
+single-process runtime becomes the blocker.
+
+DDP correctness check for TD: with batches shuffled across tokens (§P4), each DDP rank sees a random sample of tokens per batch, so gradients average correctly over the embedding matrix. Verify the gradient zeroing-for-base-rows happens *after* gradient reduction (otherwise the AllReduce would distribute the zeroed grads, which is fine but the assertion at end-of-training would need adjustment).
+
+### 14.6 Compute budget — concrete numbers
+
+Using live Apertus pilot scaling:
+
+- Layer-11 pilot: 1,020 trained tokens × 50 snippets, batch 8, elapsed
+  `1,966.9s`.
+- First full-token challenger: 17,392 trained tokens × 25 snippets. Linear
+  scaling estimate: about **4.7 wall hours**.
+- Full-token 50 snippets: about **9.3 wall hours**.
+- Full-token 100 snippets: about **18.6 wall hours**, beyond normal-partition
+  walltime for a single process.
+
+Recommend setting walltime to 8h for the first full-token 25-snippet run, or
+12h if packing a same-allocation last-layer backup.
+
+---
+
+## 15. Tools and libraries we will use
+
+| Concern | Tool | Notes |
+|---|---|---|
+| Model loading | `transformers` (pinned to uenv's 4.57.0) | has `ApertusForCausalLM` |
+| TD library code | clone of `konstantinjdobler/token-distillation` at a pinned commit | use `train_embeddings()`, `ahocorasick.py`, and `utils.py`; **skip** `tokdist.TokenDistillation` |
+| DDP / mixed precision | `accelerate` (already in our lm_eval target install) | wrap the train loop |
+| Corpus streaming | `pyarrow.parquet.iter_batches` | already battle-tested by `mix_builder.py` |
+| Tokenizer cache | `datasets.Dataset.save_to_disk` (already pulled into the uenv) | reuse the paper's caching pattern |
+| Checkpoint surgery / verification | `safetensors` + our existing `verify_hf_roundtrip.py` | mandatory post-TD check (gate §8) |
+| HF → Megatron conversion | `tools/checkpoint/convert.py` with our `loader_apertus_hf.py` | proven on R17 patch jobs 2341182/2341239/2341241 |
+| Telemetry / loss curves | TensorBoard (already wired in bakeoff_train.sbatch) | optional but cheap |
+| **Layer-suggestion probe (§6.1)** | [`AlignmentResearch/tuned-lens`](https://github.com/AlignmentResearch/tuned-lens) — Belrose et al. 2023, arXiv:2303.08112 | logit-lens is free (project residual through Apertus's existing `lm_head`); tuned-lens trains a small per-layer affine probe (~minutes per layer on one GH200). For finding a candidate `L*`, logit-lens is sufficient on first pass. If it suggests a layer materially different from the one-third-depth package suggestion, include that layer in the pilot. |
+| Multi-token Greek word inventory (for the §6.1 probe) | one-pass scan of the corpus + base tokenizer | reuse the Aho-Corasick driver from TD2; just emit the set of `(word, k)` where `k > 1` is the base-tokenizer fragment count |
+
+**Pinning the TD repo:** the official repo is small (~10 Python files) and stable. Vendor at a pinned commit hash into `subprojects/.../init_bakeoff/td/external/token_distillation/` so a moving GitHub HEAD can't break our run. Pin should land before TD0.
+
+---
+
+## 16. Open questions that still need answers
+
+Carried forward; will be resolved before TD1 lands.
+
+| # | Question | Source | How to answer |
+|---|---|---|---|
+| Q1 | Does the paper / package actually suggest testing an earlier layer? | paper §5.3 + package README | RESOLVED: yes. Paper keeps last layer as the default to avoid model-specific sweeps; package README says earlier target layers around the one-third mark can yield better results in many cases. Plan tests both. |
+| Q2 | What's the empirical coverage for our 17,408 new tokens against the 5B-Greek-token CPT corpus? | local Aho-Corasick prepass | run a one-pass coverage check before launching TD; gate as per §5 of the plan |
+| Q3 | How many tokens fragment into k > 8 base subtokens (where RoPE phase shift may be too large)? | base tokenizer applied to all 17,408 new-token surface strings | trivial offline check; report distribution of k |
+| Q4 | Does the paper sweep batch_size, or is `batch_size=1` (repo default) actually their setting? | paper appendix C / repo example.py | check arxiv:2505.20133 appendix C; example.py uses default which is `batch_size=1` — but the paper says *"We set the batch size to optimize throughput"* (p.4), so they presumably batch in practice |
+| Q5 | Does the bakeoff differentiate ReTok vs Centroid enough on §5.3 D5/D2/D7 diagnostics to gate TD on (per `SUGGESTIONS.md §7.2`)? | bakeoff arm eval | answered by reading per-arm §5.3 diagnostics after each arm's mid-training checkpoint |
+| Q6 | For polytonic TD (Phase 2), what snippet sources have enough polytonic content? | survey of available polytonic corpora | external work; descope from modern-only TD plan |
+| Q7 | Does the TD-trained checkpoint round-trip through `tools/checkpoint/convert.py` with **zero R17 drift** (since we never touched xIELU/QK-Norm during TD)? | run R17 verifier post-TD | mechanical; reuse `verify_hf_roundtrip.py` |
+| Q8 | When we combine TD (input MSE) with NTP (output CE) on Apertus's untied head, does the NTP loss act as a regularizer (paper p.7 says yes for tied) or as a confound (paper p.7 says yes for untied combination on same param)? | the paper warns about TD+NTP on the SAME param being worse; our case is TD on E + NTP on U, which is the paper's recommended pattern | implementation Q only — code already does this split correctly per `learn_output_with_ce=True` |
+| Q9 | Should the TD eval gate (§8 "Pre-CPT BPC should improve over ReTok") use a numerical threshold (e.g. >2× bootstrap stderr of ReTok)? | calibrate from V4-HF bootstrap CIs | depends on `compute_bootstrap_cis.py` results on V4 |
+| Q10 | Verify the "Kaplan et al. 2025 *Tokens to Words*" citation referenced in §6.1 before relying on the "L* ≈ first-third of layers" empirical claim | arXiv / Google Scholar search | one-off literature check; if the citation doesn't pan out, the §6.1 probe still works as a self-contained logit-lens diagnostic — we just lose the prior expectation about where L* should land |
+| Q11 | Confirm the HF hidden-state indexing convention for `target_layer = -1` on Apertus. HF returns `outputs.hidden_states[0] = embedding output` and `outputs.hidden_states[N] = output of transformer block N`. For Apertus (32 layers), `hidden_states[-1]` should be the final transformer block's output BEFORE final RMSNorm and `lm_head`. Confirm this matches what the paper means by "last layer." | quick `model.forward(..., output_hidden_states=True)` shape check on Apertus + cross-check vs `tokdist.py` reading `hidden_states[target_layer]` | mechanical; trivial offline check. If the paper means "post-final-norm representation" we'd need `target_layer = -1` to be the post-norm layer or apply the final norm manually before comparing. |
+
+---
+
+## 17. Notes from the research pass (paper + repo, 2026-05-22)
+
+Direct quotes from arxiv:2505.20133 v3 + the official repo, kept here so we don't re-derive them.
+
+**Method (paper §3.2, p.4):**
+
+> *"We learn e* by minimizing the mean-squared error between hidden states for a given target layer l: min_{e*} E_{s∼S} [ 1/|M(s_τ, s_τ*)| · Σ_{(i,j)∈M(s_τ,s_τ*)} || H_{e*}^{(l)}(s_τ*)_i − H^{(l)}(s_τ)_j ||²₂ ]. In practice, we simply use the last layer's hidden state but analyze this choice in Section 5.3."*
+
+**Hyperparameters (paper §3.3, p.4):**
+
+> *"We employ a simple setup and use the AdamW (Kingma & Ba 2017; Loshchilov & Hutter 2019) optimizer for all trainable parameters. We set the batch size to optimize throughput and run all experiments on Nvidia H100 80GB GPUs. We do not use weight decay and maintain a constant learning rate with a linear warmup. For fair comparison, we sweep for the best learning rate for all methods that require a learning rate. Since our method is aimed to serve as an initialization rather than as full-scale further training, we restrict the number of example sequences to a maximum of 25 per target token and truncate to a context length of 50 tokens. These restrictions ensure that our method is quick to run, initializing 2,500 new tokens on a single GPU in under 10 minutes."*
+
+**Output embeddings (paper §3.3, p.4):**
+
+> *"Since our method backpropagates gradients back from the hidden states, we do not learn output embeddings with our distillation-based objective. In fact, this is not possible, as our new tokens are not part of the original model that serves as the "teacher". In practice, for learning output embeddings, we can simply add a next-token prediction objective just for the output embeddings at a minimal computational overhead or freely combine our method with any other method for initializing output embeddings."*
+
+**Snippet retrieval (paper §3.3, p.4):**
+
+> *"Our main approach is to simply retrieve snippets that contain our target tokens from a domain-specific or general corpus. This can be implemented efficiently using the algorithm proposed by Aho & Corasick (1975). Then we can truncate the snippets to a small window around our target token to optimize computational efficiency."*
+
+**Freezing original embeddings (paper §5.1, p.7):**
+
+> *"Note that for Token Distillation, we only optimize new token embeddings. Therefore, we also compare against the NTP baseline, where we similarly optimize only the new token embeddings."*
+
+**Combining TD + NTP on the same parameter is worse than TD alone (paper §5.1, p.7):**
+
+> *"In support of our argument that a subtoken attention distillation objective is superior to next-token prediction for learning new token embeddings, the combination via a sum of the two objectives (Token Distillation + NTP) generally yields worse results than Token Distillation alone."*
+
+> *"However, we can add a dynamic downweighting factor (see Section 4.2) to the next-token prediction objective (Token Distillation + αNTP), which mostly alleviates the negative interference while keeping the regularizing effect."*
+
+(Implication for us: the α-NTP "regularizer" pattern is only beneficial when applied to the **input** embedding via a tied head — i.e., adds an output-loss component that backprops through the tied input/output matrix. For Apertus's *untied* setup, the equivalent is the separate CE-on-U pass we already use; we should NOT additionally add α-NTP on the input E.)
+
+**Tied-vs-untied (paper §5.1, p.7):**
+
+> *"Note that in Table 2, only for Llama-3.2-3B, Token Distillation obtains results that are on par with random initialization. Llama-3.2-3B(-i) are the only models in our lineup with tied embedding weights. Our objective does not explicitly enforce a bound on the norm of the new embedding, which in this case led to a failure mode of always generating a specific new embedding with very large norm."*
+
+(Apertus is **untied**, so this failure mode doesn't apply. Confirms the bracketing was over-cautious on the tied/untied dimension.)
+
+**Layer choice rationale (paper §5.3) — second pass:**
+
+> *"For our main experiments, we choose to keep the last layer because this choice does not necessitate a model-specific sweep over target layers. Also, the last layer is a principled choice, as it guarantees that no subtoken interactions that are only modeled in later layers are excluded from the objective."*
+
+> *"[Earlier target layers] can be much faster if we select early target layers. We leave further exploration of this as an exciting direction for future work."*
+
+(§5.3 also reports a slight downward trend in quality as the target approaches the last layer — i.e., earlier layers can marginally improve. The paper still defaults to last because the marginal gain doesn't justify a broad per-model sweep. For our one-shot Apertus adaptation, §6 now runs the smallest useful check: last layer vs the package-suggested one-third-depth layer, plus the probe-suggested `L*` only if it is materially different.)

provenance/tokenizer-selection/CHOSEN_CUTOFF.md

@@ -0,0 +1,220 @@
+# Chosen cutoff — 17,408 (curated + backfilled to alignment)
+
+**Decision date**: 2026-05-18.
+**Decided by**: user, based on the evidence in [`REPORT.md`](REPORT.md) and the
+knee-analysis plot [`artifacts/plots/knee_analysis.png`](artifacts/plots/knee_analysis.png).
+**Reviewer rounds addressed**: 2 (alignment + handoff-ambiguity).
+
+## The decision
+
+| field | value |
+|---|---|
+| **Added units** | **17,408** |
+| **Total vocab (ship)** | **148,480** = 131,072 base + 17,408 added |
+| **Alignment** | 148,480 = **128 × 1160 = 256 × 580** ✓ |
+| **Apertus front-end contract** | Preserved — Apertus base ids 0..131,071 unchanged verbatim |
+| **Curation mechanism** | **Built directly into the tokenizer.** 69 noise tokens (`02_1_5/manifests/removal_list.jsonl` in-cutoff scope) skipped during BPE merge selection; the next 69 valid merges from c3_full backfilled to maintain 17,408 useful added tokens |
+| **Cascade-skips** | **0** — none of the 17,408 backfilled merges depended on a curated merge as a sub-component |
+| **Merges walked from c3_full** | 17,477 (= 17,408 accepted + 69 skipped-for-removal) |
+| **Ship artifact** | [`variants/c3_added_17408_curated_padded/tokenizer.json`](variants/c3_added_17408_curated_padded/tokenizer.json) |
+| **Build manifest** | [`manifests/curated_padded_at_17408_manifest.json`](manifests/curated_padded_at_17408_manifest.json) |
+
+## Why the backfilled tokenizer is the canonical ship
+
+Two reviewer rounds rejected the obvious candidates:
+
+| candidate | vocab | alignment | append-only | status |
+|---|---:|:---:|:---:|---|
+| Raw `c3_added_17408` (no curation) | 148,480 | ✓ 256-aligned | ✓ preserved | metrics good, but ships 69 noise tokens |
+| Pruned `c3_added_17408_curated` (Option 2 of CURATION_REPORT) | 148,411 | ✗ NOT 128-aligned | ✗ ids renumbered | **rejected by reviewer round 2** — alignment + contract broken |
+| **Backfilled `c3_added_17408_curated_padded`** | **148,480** | **✓ 256-aligned** | **✓ preserved** | **canonical ship artifact** |
+
+The backfilled construction (built by [`scripts/01c_build_curated_backfilled.py`](scripts/01c_build_curated_backfilled.py))
+walks c3_full's merge sequence in order, **skips** the 69 in-cutoff
+ids from `02_1_5/manifests/removal_list.jsonl`, and **accepts** the
+first 17,408 surviving merges. Because BPE merges only depend on
+earlier merges as sub-components, the merge-graph validator runs
+during build — no kept merge ends up depending on a skipped one
+(measured cascade-skips at this cutoff: 0).
+
+This makes the curation **structural**, not a runtime mask: the noise
+tokens are *not in the vocab at all*, but vocab size + alignment +
+Apertus base-id preservation are intact.
+
+## Why 17,408 (the cutoff itself)
+
+From the 0 → 25,600 sweep at 1k step:
+
+| criterion | result at 17,408 |
+|---|---|
+| Greek fertility on C3_val held-out (the decision-relevant axis) | 1.345 (−44.2 % vs apertus_base 2.41) |
+| % of theoretical-max fertility improvement captured | **82.4 %** (asymptote y∞ = 1.118; full sweep at 25,600 captures 88.4 %) |
+| Marginal-Δ% per added 1k | first cutoff where the next 1k buys **< 1 %** |
+| TFG @ Apertus-55 | 0.11613 (vs apertus_base 0.11606; +0.06 % — basis points; the metric is structurally biased against script-isolated extensions, see `REPORT.md`) |
+| Apertus alignment | 17,408 = 1024 × 17; total 148,480 = 256 × 580 ✓ |
+
+**The fertility / knee analysis is the load-bearing argument.** The
+MorphScore recall bump at 17,408 (0.689 → 0.694) is real but small
+(recall is "essentially flat" 0.686–0.695 across the entire sweep);
+treat it as supporting color, not an independent pillar.
+
+The real claim: pick the cutoff where each additional 1k of vocab buys
+less than 1 % of fertility improvement — that's 17,408 — and it
+also happens to be Apertus-aligned, captures >80 % of the maximum
+achievable Greek-fertility gain, and sits at the second MorphScore
+per-word milestone (clean inflectional splits for common Greek
+nominal/verbal paradigms).
+
+## What got curated out
+
+The full `02_1_5/manifests/removal_list.jsonl` enumerates 104 noise
+tokens. In-cutoff scope at 17,408 (ids < 148,480) is **69 ids**:
+
+| removal_class | count | glossary category |
+|---|---:|---|
+| `mixed_script_artifact` | 50 | `mixed_script_token` (Greek-Latin homoglyphs + punct+Greek BPE fragments like `.Ε`, `,τι`, `/και`) |
+| `pdf_postscript_glyph` | 9 | `postscript_glyph` (PDF font-glyph residue like `/Α`, `/η`) |
+| `cleaner_linenewline_bpe_fragment` | 3 | `latin_acronym` (`LIN`, `ENEW`, `LINENEW` — BPE pieces of `LINENEWLINE`) |
+| `latin1_utf8_mojibake` | 3 | `mojibake` (`ÉÉ`-family) |
+| `cleaner_linenewline_placeholder` | 2 | `code_identifier` (`LINENEWLINE`, `NEWLINENEWLINE`) |
+| `cleaner_extraction_tag` | 2 | `latin_fragment` (`-missing`, `-decoded`) |
+| **total** | **69** | |
+
+Per-id list with `decoded` strings: [`manifests/removal_mask_at_17408.jsonl`](manifests/removal_mask_at_17408.jsonl).
+
+## Verification — directly measured (no extrapolation)
+
+### Pairwise on Apertus-55 FLORES+ (TokEval, lines config)
+
+| metric | raw 17,408 | **padded (ship)** | Δ vs raw |
+|---|---:|---:|---:|
+| TFG (Apertus-55, lower better) | 0.116128 | 0.116126 | −0.000002 |
+| Rényi-2.5 efficiency (Apertus-55) | 8.428021 | 8.427630 | −0.000391 |
+| UTF-8 completeness rate (Apertus-55) | 0.979127 | 0.979125 | −0.000002 |
+| UTF-8 char split rate (Apertus-55) | 0.010320 | 0.010320 | +0.000001 |
+| Fertility (Apertus-55, mean tokens/line) | 44.7535 | 44.7536 | +0.000093 |
+| Compression rate (Apertus-55) | 0.022345 | 0.022345 | −0.000000 |
+| Vocab utilization (Apertus-55) | 0.489278 | 0.489251 | −0.000027 |
+
+### MorphScore Greek (UD-derived, n=693)
+
+| metric | raw 17,408 | **padded (ship)** | Δ vs raw |
+|---|---:|---:|---:|
+| recall | 0.6940 | 0.6940 | 0.0000 |
+| precision | 0.6912 | 0.6912 | 0.0000 |
+| mean token-char ratio | 0.3364 | 0.3357 | −0.0007 |
+
+### In-domain Greek held-outs (02_1_3 harness on gcloud)
+
+| slice | raw 17,408 | **padded (ship)** | Δ vs raw |
+|---|---:|---:|---:|
+| C2_val — Greek word fertility | 1.3598 | 1.3591 | −0.00065 |
+| C2_test — Greek word fertility | 1.4124 | 1.4118 | −0.00060 |
+| C3_val — Greek word fertility | 1.3456 | 1.3450 | −0.00051 |
+| C3_test — Greek word fertility | 1.4011 | 1.4007 | −0.00032 |
+
+**Reading**: every directly-measured metric is either flat or **marginally
+better** for the backfilled variant. The 69 swapped-in tokens (positions
+17,409..17,477 in c3_full's merge order) fire slightly more usefully on
+real Greek than the 69 noise tokens they replaced. Direction is
+consistent across all 4 in-domain Greek slices on fertility — small
+magnitude but a real, structural win, not noise.
+
+## Embedding-budget cost
+
+| component | per-token (BF16) | total at cutoff 17,408 |
+|---|---:|---:|
+| Embedding row (4096 × 2 bytes) | 8 KiB | 17,408 × 8 KiB = **136 MiB** new |
+| LM head row (4096 × 2 bytes, `tie_word_embeddings=False`) | 8 KiB | 17,408 × 8 KiB = **136 MiB** new |
+| **Δ vs apertus_base** | 16 KiB | **272 MiB** added (= 17,408 × 16 KiB) |
+| Cumulative embed + LM head | — | 2.27 GiB (2.00 GiB Apertus base + 272 MiB added) |
+| Δ as % of full 8 B model | — | ~1.7 % |
+
+## Downstream handoff to `02_2_tokenizer_implementation`
+
+**Canonical contract:**
+
+1. Consume `variants/c3_added_17408_curated_padded/tokenizer.json`
+   verbatim. Vocab is 148,480 (256-aligned), ids 0..131,071 are exactly
+   the Apertus base, ids 131,072..148,479 are 17,408 BPE-merged Greek-
+   extension tokens with the 69 noise tokens filtered out at build
+   time.
+2. **No runtime curation logic needed.** The pruning is structural, not
+   a mask. `03_apertus_extension_and_embedding_adaptation` can warm-
+   start every added-id embedding from byte-fallback averaging without
+   any "skip these 69" branch.
+3. The mask manifest `manifests/removal_mask_at_17408.jsonl` is still
+   shipped for documentation — it tells the implementer **which 69
+   noise tokens were filtered out** so they appear in any future audit
+   trail, but it is NOT something the implementer needs to apply at
+   runtime.
+
+**Alternative (do not ship)**: the pruned `c3_added_17408_curated`
+variant remains on disk under `variants/`. It is an ablation artifact
+only — alignment-broken and append-only-broken.
+
+## Post-decision firing-count attribution
+
+After the cutoff decision, the canonical tokenizer was run over the exact
+C3 BPE training corpus (`train.parquet` plus row-aligned
+`train_manifest.parquet`) to measure how often every token fires in:
+
+1. `glossapi_nanochat_only`;
+2. `hplt_only`;
+3. `glossapi_nanochat_plus_hplt`;
+4. every exact `source_dataset`.
+
+Completed-run summary:
+
+- [`FIRING_COUNT_RUN_20260518.md`](FIRING_COUNT_RUN_20260518.md)
+- [`FIRING_COUNT_README.md`](FIRING_COUNT_README.md)
+- [`manifests/firing_count_20260518_run_summary_augmented.json`](manifests/firing_count_20260518_run_summary_augmented.json)
+
+The compact parquet bundle is published to the Apertus-extension HF repo
+under:
+
+```text
+experiments/02_1_7_intrinsic_eval_sweep_20260518/firing_counts_c3_added_17408_curated_padded/
+```
+
+Headline result: the 50/50 corpus is balanced by token mass
+(`glossapi_nanochat_only` 49.79%, HPLT 50.21%), not by row count. All
+17,408 added tokens fire in GlossAPI-nanochat; 27 are zero in HPLT; none
+are zero in the combined C3 corpus.
+
+## Build reproduction (deterministic)
+
+```bash
+cd subprojects/02_1_tokenizer_experiments/02_1_7_intrinsic_eval_sweep
+$VENV/python scripts/01c_build_curated_backfilled.py
+```
+
+The builder:
+1. Loads Apertus base (`tokenizers_local/apertus_base/tokenizer.json`)
+   and full C3 (`tokenizers_local/c3_full/tokenizer.json`, 156,672 vocab).
+2. Reads the 104-token removal list from `02_1_5/manifests/removal_list.jsonl`.
+3. Walks c3_full added merges in order; skips those whose result is in
+   the removal list and cascade-skips those depending on a skipped
+   token; accepts the first 17,408 surviving merges.
+4. Renumbers accepted merge results to ids 131,072..148,479.
+5. Writes `variants/c3_added_17408_curated_padded/tokenizer.json` and
+   the build manifest.
+
+## Pinned ship artifact
+
+```
+variants/c3_added_17408_curated_padded/tokenizer.json
+  sha256: 358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394
+  vocab:  148,480  (128-aligned ✓, 256-aligned ✓)
+  Apertus front-end contract: PRESERVED  (ids 0..131,071 verbatim)
+  Curation: 69 noise-token ids structurally absent (filtered at build)
+  Build manifest: manifests/curated_padded_at_17408_manifest.json
+```
+
+```
+variants/c3_added_17408_curated/tokenizer.json    ← ablation only, do NOT ship
+  vocab: 148,411 (not aligned, renumbered ids)
+
+variants/c3_added_17408/tokenizer.json            ← raw, no curation
+  vocab: 148,480 (aligned), but ships 69 noise tokens
+```

provenance/tokenizer-selection/CUTOFF_SWEEP_REPORT.md

@@ -0,0 +1,441 @@
+# 02_1_7 Intrinsic Eval Sweep — REPORT.md (final)
+
+> **The cutoff decision is in [`CHOSEN_CUTOFF.md`](CHOSEN_CUTOFF.md).**
+> This report is the evidence base. The ship artifact and the canonical
+> handoff to `02_2_tokenizer_implementation` are documented there.
+
+**Sweep date**: 2026-05-17 → 2026-05-18 (extended 0 → 25,600 added tokens at 1k step + curated+backfilled ship variant).
+**Tokenizers**: 33 (apertus_base + 25 raw cutoff variants + 6 curated twins + 1 curated+backfilled ship variant). The 6 curated twins + the 8 past-chosen-cutoff raw variants are kept under `variants/_archive/` as evidence-only artifacts.
+**Rows in merged parquet**: 178,658.
+**Sources**: tokeval-lines (Apertus-55), tokeval-words (Apertus-55), our_suite_02_1_3 (in-domain Greek held-outs on gcloud), morphscore (catherinearnett/morphscore Greek UD).
+**TokEval commit**: `0c4a9c641e78c8243ac753976267fd50675197cb`.
+
+## Headline plot — 4 metrics on one chart, in-domain
+
+Greek fertility on C3_val (actual in-domain held-out), TFG on
+Apertus-55 multilingual, MorphScore Greek recall, and added-vocab
+utilization on C3_val. All four shown as % improvement over
+`apertus_base`, with the sign flipped on lower-is-better metrics so
+**up always means the cutoff is doing something good**.
+
+![headline](artifacts/plots/headline_4metric_in_domain.png)
+
+## Tier 1 — in-domain Greek (02_1_3 harness on gcloud)
+
+This is the **decision-relevant** axis: actual Greek text from C2/C3
+training-distribution held-outs (anti-joined, cleaned) plus 4 latest
+glossAPI HF datasets. Not FLORES+. The 02_1_3 harness computes Greek-
+specific metrics including space-fertility (the historical fertility
+number this project has been tracking).
+
+### Greek word fertility per held-out slice
+
+| variant_id | added_tokens | curated | C2_test | C2_val | C3_test | C3_val | latest_glossAPI__archetai | latest_glossAPI__artos_zois | latest_glossAPI__eellak_articles | latest_glossAPI__opengov_deliberations_v2 |
+|---|---|---|---|---|---|---|---|---|---|---|
+| apertus_base | 0 |  | 2.5252 | 2.4228 | 2.4212 | 2.4113 | 3.2305 | 2.9122 | 2.3686 | 2.5214 |
+| add_1024 | 1024 |  | 2.1531 | 2.0890 | 2.1172 | 2.0799 | 2.6112 | 2.3456 | 1.9725 | 2.1126 |
+| add_2048 | 2048 |  | 1.9937 | 1.9315 | 1.9611 | 1.9222 | 2.4310 | 2.1844 | 1.7989 | 1.9428 |
+| add_3072 | 3072 |  | 1.8816 | 1.8231 | 1.8550 | 1.8102 | 2.3065 | 2.0641 | 1.6805 | 1.8217 |
+| add_4096 | 4096 |  | 1.8017 | 1.7408 | 1.7749 | 1.7281 | 2.2217 | 1.9848 | 1.5981 | 1.7414 |
+| add_5120 | 5120 |  | 1.7386 | 1.6752 | 1.7120 | 1.6608 | 2.1594 | 1.9278 | 1.5334 | 1.6706 |
+| add_6144 | 6144 |  | 1.6852 | 1.6238 | 1.6593 | 1.6106 | 2.1035 | 1.8698 | 1.4796 | 1.6212 |
+| add_7168 | 7168 |  | 1.6425 | 1.5795 | 1.6182 | 1.5668 | 2.0622 | 1.8320 | 1.4390 | 1.5727 |
+| add_8192 | 8192 |  | 1.6032 | 1.5413 | 1.5850 | 1.5293 | 2.0209 | 1.7983 | 1.4048 | 1.5376 |
+| add_9216 | 9216 |  | 1.5711 | 1.5112 | 1.5533 | 1.4975 | 1.9857 | 1.7663 | 1.3748 | 1.5018 |
+| add_10240 | 10240 |  | 1.5434 | 1.4859 | 1.5259 | 1.4713 | 1.9616 | 1.7426 | 1.3503 | 1.4751 |
+| add_11264 | 11264 |  | 1.5178 | 1.4635 | 1.5015 | 1.4475 | 1.9291 | 1.7188 | 1.3275 | 1.4496 |
+| add_11264_curated | 11264 | ✓ | 1.5179 | 1.4636 | 1.5018 | 1.4477 | 1.9293 |  | 1.3275 |  |
+| add_12288 | 12288 |  | 1.4959 | 1.4413 | 1.4794 | 1.4253 | 1.9078 | 1.7006 | 1.3083 | 1.4252 |
+| add_12288_curated | 12288 | ✓ | 1.4960 | 1.4414 | 1.4797 | 1.4256 | 1.9080 |  | 1.3084 |  |
+| add_13312 | 13312 |  | 1.4750 | 1.4225 | 1.4602 | 1.4061 | 1.8875 |  | 1.2912 |  |
+| add_14336 | 14336 |  | 1.4574 | 1.4042 | 1.4437 | 1.3883 | 1.8726 |  | 1.2753 |  |
+| add_15360 | 15360 |  | 1.4421 | 1.3874 | 1.4284 | 1.3723 | 1.8539 |  | 1.2629 |  |
+| add_15360_curated | 15360 | ✓ | 1.4423 | 1.3876 | 1.4288 | 1.3727 | 1.8542 |  | 1.2630 |  |
+| add_16384 | 16384 |  | 1.4259 | 1.3728 | 1.4137 | 1.3586 | 1.8413 |  | 1.2504 |  |
+| add_17408 | 17408 |  | 1.4124 | 1.3598 | 1.4011 | 1.3456 | 1.8253 |  | 1.2399 |  |
+| add_18432 | 18432 |  | 1.3998 | 1.3478 | 1.3888 | 1.3323 | 1.8131 |  | 1.2298 |  |
+| add_19456 | 19456 |  | 1.3877 | 1.3365 | 1.3778 | 1.3208 | 1.7988 |  | 1.2191 |  |
+| add_20480 | 20480 |  | 1.3776 | 1.3263 | 1.3672 | 1.3107 | 1.7859 |  | 1.2112 |  |
+| add_20480_curated | 20480 | ✓ | 1.3779 | 1.3264 | 1.3677 | 1.3112 | 1.7864 |  | 1.2114 |  |
+| add_21504 | 21504 |  | 1.3673 | 1.3165 | 1.3573 | 1.3013 | 1.7749 |  | 1.2037 |  |
+| add_22528 | 22528 |  | 1.3578 | 1.3074 | 1.3480 | 1.2915 | 1.7651 |  | 1.1945 |  |
+| add_23552 | 23552 |  | 1.3488 | 1.2986 | 1.3390 | 1.2831 | 1.7545 |  | 1.1879 |  |
+| add_24576 | 24576 |  | 1.3402 | 1.2907 | 1.3304 | 1.2748 | 1.7452 |  | 1.1819 |  |
+| add_25600 | 25600 |  | 1.3324 | 1.2826 | 1.3211 | 1.2671 | 1.7383 |  | 1.1748 |  |
+| add_25600_curated | 25600 | ✓ | 1.3328 | 1.2828 | 1.3216 | 1.2676 | 1.7388 |  | 1.1750 |  |
+
+![per-slice-fertility](artifacts/plots/fertility_per_in_domain_slice.png)
+
+### Added-vocab utilization rate per slice
+
+**The cutoff signal for diminishing returns**: when this drops, the
+added tokens stop firing on in-domain text.
+
+| variant_id | added_tokens | curated | C2_test | C2_val | C3_test | C3_val | latest_glossAPI__archetai | latest_glossAPI__artos_zois | latest_glossAPI__eellak_articles | latest_glossAPI__opengov_deliberations_v2 |
+|---|---|---|---|---|---|---|---|---|---|---|
+| add_1024 | 1024 |  | 0.9980 | 0.9971 | 0.9951 | 0.9941 | 0.9844 | 0.9844 | 0.9580 | 0.9658 |
+| add_2048 | 2048 |  | 0.9971 | 0.9956 | 0.9932 | 0.9927 | 0.9722 | 0.9736 | 0.9575 | 0.9731 |
+| add_3072 | 3072 |  | 0.9941 | 0.9925 | 0.9896 | 0.9867 | 0.9639 | 0.9619 | 0.9456 | 0.9661 |
+| add_4096 | 4096 |  | 0.9905 | 0.9883 | 0.9851 | 0.9834 | 0.9578 | 0.9529 | 0.9282 | 0.9541 |
+| add_5120 | 5120 |  | 0.9891 | 0.9871 | 0.9803 | 0.9793 | 0.9453 | 0.9406 | 0.9154 | 0.9494 |
+| add_6144 | 6144 |  | 0.9875 | 0.9842 | 0.9761 | 0.9753 | 0.9259 | 0.9248 | 0.8984 | 0.9383 |
+| add_7168 | 7168 |  | 0.9845 | 0.9801 | 0.9689 | 0.9713 | 0.9106 | 0.9083 | 0.8797 | 0.9297 |
+| add_8192 | 8192 |  | 0.9812 | 0.9757 | 0.9608 | 0.9669 | 0.8958 | 0.8933 | 0.8569 | 0.9181 |
+| add_9216 | 9216 |  | 0.9776 | 0.9712 | 0.9533 | 0.9613 | 0.8800 | 0.8760 | 0.8396 | 0.9065 |
+| add_10240 | 10240 |  | 0.9722 | 0.9648 | 0.9466 | 0.9546 | 0.8646 | 0.8610 | 0.8190 | 0.8923 |
+| add_11264 | 11264 |  | 0.9688 | 0.9579 | 0.9379 | 0.9464 | 0.8507 | 0.8475 | 0.8005 | 0.8780 |
+| add_11264_curated | 11264 | ✓ | 0.9693 | 0.9582 | 0.9384 | 0.9469 | 0.8512 |  | 0.8010 |  |
+| add_12288 | 12288 |  | 0.9631 | 0.9508 | 0.9272 | 0.9390 | 0.8367 | 0.8311 | 0.7827 | 0.8657 |
+| add_12288_curated | 12288 | ✓ | 0.9637 | 0.9512 | 0.9277 | 0.9396 | 0.8372 |  | 0.7832 |  |
+| add_13312 | 13312 |  | 0.9564 | 0.9428 | 0.9173 | 0.9305 | 0.8239 |  | 0.7620 |  |
+| add_14336 | 14336 |  | 0.9496 | 0.9356 | 0.9058 | 0.9206 | 0.8094 |  | 0.7449 |  |
+| add_15360 | 15360 |  | 0.9412 | 0.9279 | 0.8941 | 0.9113 | 0.7953 |  | 0.7277 |  |
+| add_15360_curated | 15360 | ✓ | 0.9418 | 0.9285 | 0.8948 | 0.9118 | 0.7959 |  | 0.7280 |  |
+| add_16384 | 16384 |  | 0.9343 | 0.9191 | 0.8851 | 0.9028 | 0.7825 |  | 0.7108 |  |
+| add_17408 | 17408 |  | 0.9282 | 0.9115 | 0.8755 | 0.8943 | 0.7706 |  | 0.6945 |  |
+| add_18432 | 18432 |  | 0.9190 | 0.9025 | 0.8628 | 0.8845 | 0.7584 |  | 0.6774 |  |
+| add_19456 | 19456 |  | 0.9107 | 0.8941 | 0.8516 | 0.8754 | 0.7477 |  | 0.6621 |  |
+| add_20480 | 20480 |  | 0.9006 | 0.8833 | 0.8389 | 0.8640 | 0.7364 |  | 0.6467 |  |
+| add_20480_curated | 20480 | ✓ | 0.9014 | 0.8839 | 0.8397 | 0.8644 | 0.7367 |  | 0.6468 |  |
+| add_21504 | 21504 |  | 0.8922 | 0.8738 | 0.8277 | 0.8536 | 0.7255 |  | 0.6328 |  |
+| add_22528 | 22528 |  | 0.8835 | 0.8641 | 0.8165 | 0.8426 | 0.7129 |  | 0.6187 |  |
+| add_23552 | 23552 |  | 0.8744 | 0.8535 | 0.8043 | 0.8318 | 0.7020 |  | 0.6056 |  |
+| add_24576 | 24576 |  | 0.8643 | 0.8436 | 0.7931 | 0.8218 | 0.6926 |  | 0.5928 |  |
+| add_25600 | 25600 |  | 0.8540 | 0.8337 | 0.7825 | 0.8111 | 0.6837 |  | 0.5811 |  |
+| add_25600_curated | 25600 | ✓ | 0.8547 | 0.8341 | 0.7832 | 0.8115 | 0.6841 |  | 0.5811 |  |
+
+### Chars-per-token (compression) per slice
+
+| variant_id | added_tokens | curated | C2_test | C2_val | C3_test | C3_val | latest_glossAPI__archetai | latest_glossAPI__artos_zois | latest_glossAPI__eellak_articles | latest_glossAPI__opengov_deliberations_v2 |
+|---|---|---|---|---|---|---|---|---|---|---|
+| apertus_base | 0 |  | 2.5272 | 2.6307 | 2.5220 | 2.6093 | 1.9618 | 2.1190 | 2.7354 | 2.4872 |
+| add_1024 | 1024 |  | 2.8957 | 2.9795 | 2.8053 | 2.9632 | 2.2963 | 2.5023 | 3.1155 | 2.8748 |
+| add_2048 | 2048 |  | 3.0882 | 3.1770 | 2.9767 | 3.1678 | 2.4146 | 2.6379 | 3.3165 | 3.0769 |
+| add_3072 | 3072 |  | 3.2389 | 3.3283 | 3.1048 | 3.3312 | 2.5037 | 2.7476 | 3.4750 | 3.2358 |
+| add_4096 | 4096 |  | 3.3559 | 3.4528 | 3.2086 | 3.4609 | 2.5693 | 2.8259 | 3.5868 | 3.3503 |
+| add_5120 | 5120 |  | 3.4543 | 3.5596 | 3.2951 | 3.5760 | 2.6185 | 2.8843 | 3.6816 | 3.4577 |
+| add_6144 | 6144 |  | 3.5429 | 3.6486 | 3.3721 | 3.6676 | 2.6660 | 2.9475 | 3.7676 | 3.5373 |
+| add_7168 | 7168 |  | 3.6167 | 3.7275 | 3.4344 | 3.7508 | 2.7028 | 2.9907 | 3.8373 | 3.6195 |
+| add_8192 | 8192 |  | 3.6871 | 3.7988 | 3.4864 | 3.8250 | 2.7385 | 3.0285 | 3.8953 | 3.6810 |
+| add_9216 | 9216 |  | 3.7470 | 3.8584 | 3.5380 | 3.8900 | 2.7696 | 3.0650 | 3.9468 | 3.7473 |
+| add_10240 | 10240 |  | 3.8004 | 3.9087 | 3.5828 | 3.9460 | 2.7910 | 3.0940 | 3.9901 | 3.7973 |
+| add_11264 | 11264 |  | 3.8506 | 3.9550 | 3.6242 | 3.9977 | 2.8219 | 3.1247 | 4.0310 | 3.8464 |
+| add_11264_curated | 11264 | ✓ | 3.8476 | 3.9516 | 3.6189 | 3.9933 | 2.8197 |  | 4.0278 |  |
+| add_12288 | 12288 |  | 3.8950 | 4.0015 | 3.6628 | 4.0480 | 2.8423 | 3.1476 | 4.0669 | 3.8945 |
+| add_12288_curated | 12288 | ✓ | 3.8918 | 3.9980 | 3.6573 | 4.0433 | 2.8401 |  | 4.0635 |  |
+| add_13312 | 13312 |  | 3.9379 | 4.0412 | 3.6963 | 4.0918 | 2.8629 |  | 4.0987 |  |
+| add_14336 | 14336 |  | 3.9747 | 4.0811 | 3.7261 | 4.1332 | 2.8769 |  | 4.1289 |  |
+| add_15360 | 15360 |  | 4.0083 | 4.1188 | 3.7552 | 4.1711 | 2.8956 |  | 4.1534 |  |
+| add_15360_curated | 15360 | ✓ | 4.0046 | 4.1143 | 3.7493 | 4.1657 | 2.8931 |  | 4.1495 |  |
+| add_16384 | 16384 |  | 4.0438 | 4.1516 | 3.7828 | 4.2050 | 2.9085 |  | 4.1785 |  |
+| add_17408 | 17408 |  | 4.0742 | 4.1816 | 3.8070 | 4.2375 | 2.9254 |  | 4.1999 |  |
+| add_18432 | 18432 |  | 4.1028 | 4.2097 | 3.8310 | 4.2705 | 2.9380 |  | 4.2235 |  |
+| add_19456 | 19456 |  | 4.1306 | 4.2374 | 3.8536 | 4.3001 | 2.9528 |  | 4.2455 |  |
+| add_20480 | 20480 |  | 4.1545 | 4.2622 | 3.8750 | 4.3267 | 2.9655 |  | 4.2617 |  |
+| add_20480_curated | 20480 | ✓ | 4.1499 | 4.2563 | 3.8684 | 4.3201 | 2.9624 |  | 4.2572 |  |
+| add_21504 | 21504 |  | 4.1784 | 4.2855 | 3.8943 | 4.3511 | 2.9782 |  | 4.2775 |  |
+| add_22528 | 22528 |  | 4.2010 | 4.3071 | 3.9129 | 4.3768 | 2.9885 |  | 4.2972 |  |
+| add_23552 | 23552 |  | 4.2227 | 4.3289 | 3.9320 | 4.3993 | 2.9995 |  | 4.3192 |  |
+| add_24576 | 24576 |  | 4.2438 | 4.3490 | 3.9494 | 4.4223 | 3.0097 |  | 4.3320 |  |
+| add_25600 | 25600 |  | 4.2630 | 4.3692 | 3.9684 | 4.4437 | 3.0180 |  | 4.3479 |  |
+| add_25600_curated | 25600 | ✓ | 4.2576 | 4.3627 | 3.9614 | 4.4366 | 3.0147 |  | 4.3430 |  |
+
+## Tier 1 — paper-aligned (TokEval on Apertus-55 FLORES+)
+
+These match the eval surface area the Apertus paper §2.2 uses; cite
+for cross-paper comparison (Meister 2025 / Foroutan et al. 2025a).
+
+### TFG (Apertus-55 multilingual fairness)
+
+**Lower = more fair.** U-shape with minimum at ~4k — the only metric
+with a clear non-monotonic optimum on this run.
+
+| variant_id | added_tokens | curated | value |
+|---|---|---|---|
+| apertus_base | 0 |  | 0.1161 |
+| add_1024 | 1024 |  | 0.1150 |
+| add_2048 | 2048 |  | 0.1146 |
+| add_3072 | 3072 |  | 0.1144 |
+| add_4096 | 4096 |  | 0.1144 |
+| add_5120 | 5120 |  | 0.1145 |
+| add_6144 | 6144 |  | 0.1146 |
+| add_7168 | 7168 |  | 0.1148 |
+| add_8192 | 8192 |  | 0.1149 |
+| add_9216 | 9216 |  | 0.1151 |
+| add_10240 | 10240 |  | 0.1152 |
+| add_11264 | 11264 |  | 0.1154 |
+| add_11264_curated | 11264 | ✓ | 0.1154 |
+| add_12288 | 12288 |  | 0.1155 |
+| add_12288_curated | 12288 | ✓ | 0.1155 |
+| add_13312 | 13312 |  | 0.1157 |
+| add_14336 | 14336 |  | 0.1158 |
+| add_15360 | 15360 |  | 0.1159 |
+| add_15360_curated | 15360 | ✓ | 0.1159 |
+| add_16384 | 16384 |  | 0.1160 |
+| add_17408 | 17408 |  | 0.1161 |
+| add_18432 | 18432 |  | 0.1162 |
+| add_19456 | 19456 |  | 0.1163 |
+| add_20480 | 20480 |  | 0.1164 |
+| add_20480_curated | 20480 | ✓ | 0.1164 |
+| add_21504 | 21504 |  | 0.1165 |
+| add_22528 | 22528 |  | 0.1166 |
+| add_23552 | 23552 |  | 0.1167 |
+| add_24576 | 24576 |  | 0.1168 |
+| add_25600 | 25600 |  | 0.1169 |
+| add_25600_curated | 25600 | ✓ | 0.1169 |
+
+### Fertility (FLORES+ Greek, words config)
+
+Same shape as the in-domain fertility above but on cleaner narrower
+data — values are lower (clean Greek is easier to tokenize).
+
+| variant_id | added_tokens | curated | value |
+|---|---|---|---|
+| apertus_base | 0 |  | 2.2041 |
+| add_1024 | 1024 |  | 1.9825 |
+| add_2048 | 2048 |  | 1.8545 |
+| add_3072 | 3072 |  | 1.7639 |
+| add_4096 | 4096 |  | 1.7003 |
+| add_5120 | 5120 |  | 1.6408 |
+| add_6144 | 6144 |  | 1.5989 |
+| add_7168 | 7168 |  | 1.5617 |
+| add_8192 | 8192 |  | 1.5309 |
+| add_9216 | 9216 |  | 1.5065 |
+| add_10240 | 10240 |  | 1.4809 |
+| add_11264 | 11264 |  | 1.4599 |
+| add_11264_curated | 11264 | ✓ | 1.4607 |
+| add_12288 | 12288 |  | 1.4417 |
+| add_12288_curated | 12288 | ✓ | 1.4426 |
+| add_13312 | 13312 |  | 1.4228 |
+| add_14336 | 14336 |  | 1.4076 |
+| add_15360 | 15360 |  | 1.3941 |
+| add_15360_curated | 15360 | ✓ | 1.3951 |
+| add_16384 | 16384 |  | 1.3816 |
+| add_17408 | 17408 |  | 1.3693 |
+| add_18432 | 18432 |  | 1.3582 |
+| add_19456 | 19456 |  | 1.3481 |
+| add_20480 | 20480 |  | 1.3399 |
+| add_20480_curated | 20480 | ✓ | 1.3408 |
+| add_21504 | 21504 |  | 1.3317 |
+| add_22528 | 22528 |  | 1.3225 |
+| add_23552 | 23552 |  | 1.3146 |
+| add_24576 | 24576 |  | 1.3076 |
+| add_25600 | 25600 |  | 1.3008 |
+| add_25600_curated | 25600 | ✓ | 1.3018 |
+
+## Tier 2 — multi-criteria supplement
+
+### MorphScore Greek (catherinearnett/morphscore)
+
+21,428 Greek words from UD Greek-GDT with gold morpheme boundaries;
+`n=693` after the freq-scale + length filter. **Recall** = fraction of
+gold morpheme boundaries the tokenizer hits; **precision** = fraction
+of tokenizer boundaries that hit a real morpheme boundary; **token-
+char ratio** = mean tokens-per-character (lower = less over-segmenting).
+
+| variant_id | added_tokens | curated | macro_f1 | macro_precision | macro_recall | mean_token_char_ratio | micro_f1 | micro_precision | micro_recall | morphscore_precision | morphscore_precision_std | morphscore_recall | morphscore_recall_std | num_samples |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| apertus_base | 0 |  | 0.0210 | 0.0152 | 0.0339 | 0.4871 | 0.0217 | 0.0160 | 0.0338 | 0.6829 | 0.0034 | 0.6979 | 0.0034 | 693.0000 |
+| add_1024 | 1024 |  | 0.0179 | 0.0133 | 0.0274 | 0.4426 | 0.0189 | 0.0144 | 0.0274 | 0.6823 | 0.0034 | 0.6940 | 0.0034 | 693.0000 |
+| add_2048 | 2048 |  | 0.0126 | 0.0095 | 0.0188 | 0.4213 | 0.0133 | 0.0103 | 0.0187 | 0.6812 | 0.0034 | 0.6890 | 0.0034 | 693.0000 |
+| add_3072 | 3072 |  | 0.0130 | 0.0099 | 0.0188 | 0.4073 | 0.0136 | 0.0107 | 0.0187 | 0.6813 | 0.0034 | 0.6890 | 0.0034 | 693.0000 |
+| add_4096 | 4096 |  | 0.0125 | 0.0096 | 0.0180 | 0.3944 | 0.0134 | 0.0107 | 0.0180 | 0.6816 | 0.0034 | 0.6890 | 0.0034 | 693.0000 |
+| add_5120 | 5120 |  | 0.0116 | 0.0089 | 0.0166 | 0.3833 | 0.0126 | 0.0101 | 0.0166 | 0.6817 | 0.0034 | 0.6883 | 0.0034 | 693.0000 |
+| add_6144 | 6144 |  | 0.0114 | 0.0089 | 0.0159 | 0.3730 | 0.0122 | 0.0100 | 0.0158 | 0.6816 | 0.0034 | 0.6872 | 0.0034 | 693.0000 |
+| add_7168 | 7168 |  | 0.0109 | 0.0085 | 0.0152 | 0.3669 | 0.0118 | 0.0097 | 0.0151 | 0.6815 | 0.0034 | 0.6870 | 0.0034 | 693.0000 |
+| add_8192 | 8192 |  | 0.0100 | 0.0078 | 0.0137 | 0.3637 | 0.0107 | 0.0088 | 0.0137 | 0.6812 | 0.0034 | 0.6859 | 0.0034 | 693.0000 |
+| add_9216 | 9216 |  | 0.0100 | 0.0078 | 0.0137 | 0.3588 | 0.0108 | 0.0090 | 0.0137 | 0.6814 | 0.0034 | 0.6861 | 0.0034 | 693.0000 |
+| add_10240 | 10240 |  | 0.0101 | 0.0079 | 0.0137 | 0.3546 | 0.0109 | 0.0091 | 0.0137 | 0.6868 | 0.0034 | 0.6915 | 0.0034 | 693.0000 |
+| add_11264 | 11264 |  | 0.0095 | 0.0075 | 0.0130 | 0.3507 | 0.0104 | 0.0087 | 0.0130 | 0.6866 | 0.0034 | 0.6909 | 0.0034 | 693.0000 |
+| add_11264_curated | 11264 | ✓ | 0.0095 | 0.0075 | 0.0130 | 0.3507 | 0.0104 | 0.0087 | 0.0130 | 0.6866 | 0.0034 | 0.6909 | 0.0034 | 693.0000 |
+| add_12288 | 12288 |  | 0.0096 | 0.0076 | 0.0130 | 0.3478 | 0.0105 | 0.0088 | 0.0130 | 0.6866 | 0.0034 | 0.6909 | 0.0034 | 693.0000 |
+| add_12288_curated | 12288 | ✓ | 0.0096 | 0.0076 | 0.0130 | 0.3478 | 0.0105 | 0.0088 | 0.0130 | 0.6866 | 0.0034 | 0.6909 | 0.0034 | 693.0000 |
+| add_13312 | 13312 |  | 0.0090 | 0.0071 | 0.0123 | 0.3465 | 0.0099 | 0.0083 | 0.0122 | 0.6866 | 0.0034 | 0.6908 | 0.0034 | 693.0000 |
+| add_14336 | 14336 |  | 0.0090 | 0.0071 | 0.0123 | 0.3454 | 0.0099 | 0.0083 | 0.0122 | 0.6867 | 0.0034 | 0.6909 | 0.0034 | 693.0000 |
+| add_15360 | 15360 |  | 0.0079 | 0.0063 | 0.0108 | 0.3419 | 0.0088 | 0.0074 | 0.0108 | 0.6863 | 0.0034 | 0.6892 | 0.0034 | 693.0000 |
+| add_15360_curated | 15360 | ✓ | 0.0079 | 0.0063 | 0.0108 | 0.3419 | 0.0088 | 0.0074 | 0.0108 | 0.6863 | 0.0034 | 0.6892 | 0.0034 | 693.0000 |
+| add_16384 | 16384 |  | 0.0081 | 0.0065 | 0.0108 | 0.3393 | 0.0088 | 0.0075 | 0.0108 | 0.6864 | 0.0034 | 0.6892 | 0.0034 | 693.0000 |
+| add_17408 | 17408 |  | 0.0081 | 0.0065 | 0.0108 | 0.3364 | 0.0089 | 0.0076 | 0.0108 | 0.6912 | 0.0034 | 0.6940 | 0.0034 | 693.0000 |
+| add_18432 | 18432 |  | 0.0083 | 0.0067 | 0.0108 | 0.3339 | 0.0089 | 0.0076 | 0.0108 | 0.6912 | 0.0034 | 0.6940 | 0.0034 | 693.0000 |
+| add_19456 | 19456 |  | 0.0078 | 0.0064 | 0.0101 | 0.3311 | 0.0084 | 0.0072 | 0.0101 | 0.6922 | 0.0034 | 0.6947 | 0.0034 | 693.0000 |
+| add_20480 | 20480 |  | 0.0078 | 0.0064 | 0.0101 | 0.3298 | 0.0084 | 0.0072 | 0.0101 | 0.6922 | 0.0034 | 0.6947 | 0.0034 | 693.0000 |
+| add_20480_curated | 20480 | ✓ | 0.0078 | 0.0064 | 0.0101 | 0.3298 | 0.0084 | 0.0072 | 0.0101 | 0.6922 | 0.0034 | 0.6947 | 0.0034 | 693.0000 |
+| add_21504 | 21504 |  | 0.0078 | 0.0064 | 0.0101 | 0.3282 | 0.0084 | 0.0072 | 0.0101 | 0.6924 | 0.0034 | 0.6949 | 0.0034 | 693.0000 |
+| add_22528 | 22528 |  | 0.0078 | 0.0064 | 0.0101 | 0.3265 | 0.0084 | 0.0073 | 0.0101 | 0.6924 | 0.0034 | 0.6949 | 0.0034 | 693.0000 |
+| add_23552 | 23552 |  | 0.0078 | 0.0064 | 0.0101 | 0.3244 | 0.0085 | 0.0073 | 0.0101 | 0.6926 | 0.0034 | 0.6950 | 0.0034 | 693.0000 |
+| add_24576 | 24576 |  | 0.0079 | 0.0065 | 0.0101 | 0.3240 | 0.0085 | 0.0073 | 0.0101 | 0.6926 | 0.0034 | 0.6950 | 0.0034 | 693.0000 |
+| add_25600 | 25600 |  | 0.0079 | 0.0065 | 0.0101 | 0.3225 | 0.0085 | 0.0074 | 0.0101 | 0.6926 | 0.0034 | 0.6950 | 0.0034 | 693.0000 |
+| add_25600_curated | 25600 | ✓ | 0.0079 | 0.0065 | 0.0101 | 0.3225 | 0.0085 | 0.0074 | 0.0101 | 0.6926 | 0.0034 | 0.6950 | 0.0034 | 693.0000 |
+
+![morphscore](artifacts/plots/morphscore_greek.png)
+
+**Reading**: recall is essentially flat (~0.69) across all cutoffs —
+the added vocabulary does not appreciably improve morpheme-boundary
+alignment. Precision improves marginally (+0.5 % at 11k). The
+**token-char ratio drops 29 %** (0.49 → 0.35), confirming the added
+vocab makes Greek encoding more compact without dramatically improving
+morphological structure capture. Useful nuance the 4 paper metrics
+don't see.
+
+### Rényi-2.5 efficiency (Apertus-55 global)
+
+| variant_id | added_tokens | curated | value |
+|---|---|---|---|
+| apertus_base | 0 |  | 8.4483 |
+| add_1024 | 1024 |  | 8.4429 |
+| add_2048 | 2048 |  | 8.4400 |
+| add_3072 | 3072 |  | 8.4377 |
+| add_4096 | 4096 |  | 8.4362 |
+| add_5120 | 5120 |  | 8.4348 |
+| add_6144 | 6144 |  | 8.4338 |
+| add_7168 | 7168 |  | 8.4329 |
+| add_8192 | 8192 |  | 8.4321 |
+| add_9216 | 9216 |  | 8.4315 |
+| add_10240 | 10240 |  | 8.4308 |
+| add_11264 | 11264 |  | 8.4303 |
+| add_11264_curated | 11264 | ✓ | 8.4299 |
+| add_12288 | 12288 |  | 8.4298 |
+| add_12288_curated | 12288 | ✓ | 8.4295 |
+| add_13312 | 13312 |  | 8.4294 |
+| add_14336 | 14336 |  | 8.4290 |
+| add_15360 | 15360 |  | 8.4286 |
+| add_15360_curated | 15360 | ✓ | 8.4283 |
+| add_16384 | 16384 |  | 8.4283 |
+| add_17408 | 17408 |  | 8.4280 |
+| add_18432 | 18432 |  | 8.4277 |
+| add_19456 | 19456 |  | 8.4275 |
+| add_20480 | 20480 |  | 8.4273 |
+| add_20480_curated | 20480 | ✓ | 8.4269 |
+| add_21504 | 21504 |  | 8.4271 |
+| add_22528 | 22528 |  | 8.4268 |
+| add_23552 | 23552 |  | 8.4266 |
+| add_24576 | 24576 |  | 8.4264 |
+| add_25600 | 25600 |  | 8.4263 |
+| add_25600_curated | 25600 | ✓ | 8.4259 |
+
+### UTF-8 completeness (Apertus-55 global)
+
+| variant_id | added_tokens | curated | value |
+|---|---|---|---|
+| apertus_base | 0 |  | 0.9792 |
+| add_1024 | 1024 |  | 0.9792 |
+| add_2048 | 2048 |  | 0.9792 |
+| add_3072 | 3072 |  | 0.9792 |
+| add_4096 | 4096 |  | 0.9792 |
+| add_5120 | 5120 |  | 0.9792 |
+| add_6144 | 6144 |  | 0.9791 |
+| add_7168 | 7168 |  | 0.9791 |
+| add_8192 | 8192 |  | 0.9792 |
+| add_9216 | 9216 |  | 0.9791 |
+| add_10240 | 10240 |  | 0.9791 |
+| add_11264 | 11264 |  | 0.9791 |
+| add_11264_curated | 11264 | ✓ | 0.9791 |
+| add_12288 | 12288 |  | 0.9791 |
+| add_12288_curated | 12288 | ✓ | 0.9791 |
+| add_13312 | 13312 |  | 0.9791 |
+| add_14336 | 14336 |  | 0.9791 |
+| add_15360 | 15360 |  | 0.9791 |
+| add_15360_curated | 15360 | ✓ | 0.9791 |
+| add_16384 | 16384 |  | 0.9791 |
+| add_17408 | 17408 |  | 0.9791 |
+| add_18432 | 18432 |  | 0.9791 |
+| add_19456 | 19456 |  | 0.9791 |
+| add_20480 | 20480 |  | 0.9791 |
+| add_20480_curated | 20480 | ✓ | 0.9791 |
+| add_21504 | 21504 |  | 0.9791 |
+| add_22528 | 22528 |  | 0.9791 |
+| add_23552 | 23552 |  | 0.9791 |
+| add_24576 | 24576 |  | 0.9791 |
+| add_25600 | 25600 |  | 0.9791 |
+| add_25600_curated | 25600 | ✓ | 0.9791 |
+
+## Curated-arm delta (consumes 02_1_5 removal_list)
+
+Curated twins remove 39 (at 11,264) / 44 (at 12,288) tokens after
+merge-graph validation. **In every metric tested, the delta vs the
+un-curated variant is essentially zero** — 39/44 token removal is free.
+
+### Cutoff 11,264
+
+| metric | source | raw | curated | Δ |
+|---|---|---|---|---|
+| greek_word_space_fertility | our_suite_02_1_3 | 1.5444 | 1.5313 | -0.0131 |
+| chars_per_token | our_suite_02_1_3 | 3.6564 | 3.7098 | 0.0534 |
+| eval_added_vocab_utilization_rate | our_suite_02_1_3 | 0.8985 | 0.9108 | 0.0124 |
+| tokenizer_fairness_gini | tokeval-lines | 0.1154 | 0.1154 | -0.0000 |
+| renyi_efficiency | tokeval-lines | 8.4303 | 8.4299 | -0.0004 |
+| utf8_token_integrity | tokeval-lines | 0.9791 | 0.9791 | 0.0000 |
+| morphscore_recall | morphscore | 0.6909 | 0.6909 | 0.0000 |
+| morphscore_precision | morphscore | 0.6866 | 0.6866 | 0.0000 |
+| mean_token_char_ratio | morphscore | 0.3507 | 0.3507 | 0.0000 |
+
+### Cutoff 12,288
+
+| metric | source | raw | curated | Δ |
+|---|---|---|---|---|
+| greek_word_space_fertility | our_suite_02_1_3 | 1.5230 | 1.5098 | -0.0131 |
+| chars_per_token | our_suite_02_1_3 | 3.6948 | 3.7490 | 0.0542 |
+| eval_added_vocab_utilization_rate | our_suite_02_1_3 | 0.8870 | 0.9004 | 0.0134 |
+| tokenizer_fairness_gini | tokeval-lines | 0.1155 | 0.1155 | -0.0000 |
+| renyi_efficiency | tokeval-lines | 8.4298 | 8.4295 | -0.0004 |
+| utf8_token_integrity | tokeval-lines | 0.9791 | 0.9791 | -0.0000 |
+| morphscore_recall | morphscore | 0.6909 | 0.6909 | 0.0000 |
+| morphscore_precision | morphscore | 0.6866 | 0.6866 | 0.0000 |
+| mean_token_char_ratio | morphscore | 0.3478 | 0.3478 | 0.0000 |
+
+## Methodology delta — TokEval-fertility vs 02_1_3-fertility on Greek
+
+Same 15 tokenizers, same target (Greek), different eval sets and
+different normalizations. **TokEval = FLORES+ ell_Grek, words config.**
+**02_1_3 = our in-domain held-outs, greek_word_space_fertility.**
+
+Use this to decide which fertility number to cite — TokEval for paper
+comparability, 02_1_3 for in-domain decision-making.
+
+| variant_id | added_tokens | TokEval (FLORES+ Greek) | 02_1_3 (C3_val) |
+|---|---|---|---|
+| apertus_base | 0 | 2.2041 | 2.4113 |
+| add_1024 | 1024 | 1.9825 | 2.0799 |
+| add_2048 | 2048 | 1.8545 | 1.9222 |
+| add_3072 | 3072 | 1.7639 | 1.8102 |
+| add_4096 | 4096 | 1.7003 | 1.7281 |
+| add_5120 | 5120 | 1.6408 | 1.6608 |
+| add_6144 | 6144 | 1.5989 | 1.6106 |
+| add_7168 | 7168 | 1.5617 | 1.5668 |
+| add_8192 | 8192 | 1.5309 | 1.5293 |
+| add_9216 | 9216 | 1.5065 | 1.4975 |
+| add_10240 | 10240 | 1.4809 | 1.4713 |
+| add_11264 | 11264 | 1.4599 | 1.4475 |
+| add_11264_curated | 11264 | 1.4607 | 1.4477 |
+| add_12288 | 12288 | 1.4417 | 1.4253 |
+| add_12288_curated | 12288 | 1.4426 | 1.4256 |
+| add_13312 | 13312 | 1.4228 | 1.4061 |
+| add_14336 | 14336 | 1.4076 | 1.3883 |
+| add_15360 | 15360 | 1.3941 | 1.3723 |
+| add_15360_curated | 15360 | 1.3951 | 1.3727 |
+| add_16384 | 16384 | 1.3816 | 1.3586 |
+| add_17408 | 17408 | 1.3693 | 1.3456 |
+| add_18432 | 18432 | 1.3582 | 1.3323 |
+| add_19456 | 19456 | 1.3481 | 1.3208 |
+| add_20480 | 20480 | 1.3399 | 1.3107 |
+| add_20480_curated | 20480 | 1.3408 | 1.3112 |
+| add_21504 | 21504 | 1.3317 | 1.3013 |
+| add_22528 | 22528 | 1.3225 | 1.2915 |
+| add_23552 | 23552 | 1.3146 | 1.2831 |
+| add_24576 | 24576 | 1.3076 | 1.2748 |
+| add_25600 | 25600 | 1.3008 | 1.2671 |
+| add_25600_curated | 25600 | 1.3018 | 1.2676 |
+
+## Reproduction
+
+```bash
+cd subprojects/02_1_tokenizer_experiments/02_1_7_intrinsic_eval_sweep
+$VENV/python scripts/01_build_variants_inline.py    # local 13 variants
+$VENV/python scripts/01b_build_curated_variants.py  # 2 curated twins
+$VENV/python scripts/02_prep_eval_configs.py
+bash scripts/03a_run_tokeval.sh                     # TokEval 2 jobs (~30 min)
+# gcloud: resume → run 02_1_3 harness for our 4 in-house slices → suspend
+$VENV/python scripts/07_morphscore_greek.py         # MorphScore Greek
+$VENV/python scripts/08_merge_all.py                # merge all 4 sources
+$VENV/python scripts/09_render_final_report.py      # this file + plots
+```

provenance/tokenizer-selection/curated_padded_at_17408_manifest.json

@@ -0,0 +1,15 @@
+{
+  "variant_id": "add_17408_curated_padded",
+  "added_tokens": 17408,
+  "total_vocab": 148480,
+  "alignment_128": true,
+  "alignment_256": true,
+  "apertus_base_preserved_verbatim": true,
+  "removal_list_source": "/home/foivos/Projects/glossapi-tokenizer-extension/subprojects/02_1_tokenizer_experiments/02_1_5_added_token_curation/manifests/removal_list.jsonl",
+  "removal_list_size_full": 104,
+  "merges_walked_from_c3_full": 17477,
+  "merges_skipped_for_removal": 69,
+  "merges_skipped_for_cascade": 0,
+  "merges_accepted": 17408,
+  "ship_artifact_path": "/home/foivos/Projects/glossapi-tokenizer-extension/subprojects/02_1_tokenizer_experiments/02_1_7_intrinsic_eval_sweep/variants/c3_added_17408_curated_padded/tokenizer.json"
+}

results/SUMMARY.md

@@ -0,0 +1,91 @@
+# 3.5B continuation results - Vanilla vs ReTok vs TD layer11
+
+Generated UTC: `2026-05-25T05:23:53+00:00`.
+
+This summarizes the continuation run `continuation_3p5b_20260524T143012Z`,
+which extended Vanilla, ReTok, and TD layer11 from iter 476 (~2.0B tokens)
+to iter 834 (~3.5B tokens). Local JSON snapshots live under
+`per_iter_results/`; remote full artifacts remain on Clariden under
+`/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_*`.
+
+## Bottom line
+
+- TD layer11 is the best final benchmark arm overall: it is first on English
+  retention and multilingual aggregates, and narrowly first on the Greek
+  aggregate at iter 834.
+- Vanilla still has the best tokenizer-fair heldout Greek BPC, but its
+  downstream Greek aggregate declined during the 2.0B -> 3.5B continuation.
+- ReTok improves fastest on BPC and wins Greek MMLU / INCLUDE-44 Greek at
+  iter 834, but it remains behind TD and Vanilla on the Greek aggregate.
+- If selecting for the actual downstream bakeoff objective, TD layer11 is now
+  the leading candidate. If selecting only for heldout BPC, Vanilla remains
+  ahead.
+
+## Aggregate scoreboard at iter 834
+
+| Arm | Greek agg | Delta vs 476 | EN retention | Delta vs 476 | Multilingual | Delta vs 476 | BPC lower better | BPC delta |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| Vanilla | 0.4339 | -0.70 pp | 0.6782 | -0.36 pp | 0.4923 | +0.22 pp | 0.4724 | -0.0182 |
+| ReTok | 0.4246 | +0.96 pp | 0.6786 | +0.36 pp | 0.4864 | -0.09 pp | 0.5390 | -0.0349 |
+| TD layer11 | 0.4344 | +0.90 pp | 0.6865 | +0.38 pp | 0.4967 | +0.68 pp | 0.5054 | -0.0256 |
+
+## Per-task winners at iter 834
+
+| Group | Task | Vanilla | ReTok | TD layer11 | Winner |
+|---|---|---:|---:|---:|---|
+| EN retention | `mmlu` | 0.5330 | 0.5565 | 0.5556 | ReTok |
+| EN retention | `hellaswag` | 0.7573 | 0.7556 | 0.7632 | TD layer11 |
+| EN retention | `arc_easy` | 0.7866 | 0.7576 | 0.7837 | Vanilla |
+| EN retention | `arc_challenge` | 0.5230 | 0.5247 | 0.5358 | TD layer11 |
+| EN retention | `piqa` | 0.7927 | 0.7851 | 0.7894 | Vanilla |
+| EN retention | `winogrande` | 0.6764 | 0.6922 | 0.6914 | ReTok |
+| Multilingual | `global_mmlu` | 0.4505 | 0.4603 | 0.4628 | TD layer11 |
+| Multilingual | `xcopa` | 0.6178 | 0.5920 | 0.6156 | Vanilla |
+| Multilingual | `xnli` | 0.4087 | 0.4070 | 0.4117 | TD layer11 |
+| Greek | `global_mmlu_full_el` | 0.4145 | 0.4153 | 0.4036 | ReTok |
+| Greek | `include_base_44_greek_few_shot_en` | 0.4022 | 0.4058 | 0.4004 | ReTok |
+| Greek | `belebele_ell_Grek` | 0.5078 | 0.4933 | 0.5389 | TD layer11 |
+| Greek | `arc_challenge_mt_el` | 0.4130 | 0.3899 | 0.4061 | Vanilla |
+| Greek | `xnli_el` | 0.3831 | 0.3695 | 0.3851 | TD layer11 |
+| Greek | `xquad_el` | 0.2868 | 0.3084 | 0.3364 | TD layer11 |
+| Greek | `global_piqa_completions_ell_grek` | 0.6300 | 0.5900 | 0.5700 | Vanilla |
+
+## Change from iter 476 to iter 834
+
+| Group | Task | Vanilla delta | ReTok delta | TD layer11 delta |
+|---|---|---:|---:|---:|
+| EN retention | `mmlu` | -0.09 pp | +0.24 pp | +0.56 pp |
+| EN retention | `hellaswag` | -0.21 pp | +0.69 pp | +0.27 pp |
+| EN retention | `arc_easy` | +0.13 pp | -0.59 pp | +0.29 pp |
+| EN retention | `arc_challenge` | -0.43 pp | +0.94 pp | +0.85 pp |
+| EN retention | `piqa` | -0.22 pp | -0.11 pp | -0.27 pp |
+| EN retention | `winogrande` | -1.34 pp | +1.03 pp | +0.55 pp |
+| Multilingual | `global_mmlu` | -0.64 pp | +0.14 pp | +1.34 pp |
+| Multilingual | `xcopa` | +1.09 pp | -0.71 pp | +0.42 pp |
+| Multilingual | `xnli` | +0.22 pp | +0.30 pp | +0.28 pp |
+| Greek | `global_mmlu_full_el` | -0.69 pp | +1.62 pp | +1.77 pp |
+| Greek | `include_base_44_greek_few_shot_en` | -1.63 pp | +3.26 pp | -0.36 pp |
+| Greek | `belebele_ell_Grek` | -0.56 pp | -0.33 pp | +1.11 pp |
+| Greek | `arc_challenge_mt_el` | -0.77 pp | +1.79 pp | +0.26 pp |
+| Greek | `xnli_el` | -1.89 pp | -0.56 pp | +0.48 pp |
+| Greek | `xquad_el` | -2.33 pp | -0.08 pp | +1.02 pp |
+| Greek | `global_piqa_completions_ell_grek` | +3.00 pp | +1.00 pp | +2.00 pp |
+
+## New-token diagnostics
+
+Measured on the 500-document heldout slice. Top-k is the fraction of
+positions whose correct new token appears in the model's top-k predictions;
+lower mean rank is better.
+
+| Arm | Top-1 at new target | Delta from 585 | Top-5 at new target | Delta from 585 | Mean rank | Delta from 585 |
+|---|---:|---:|---:|---:|---:|---:|
+| ReTok | 0.3799 | +2.00 pp | 0.5469 | +2.11 pp | 228.9 | -31.4 |
+| TD layer11 | 0.4105 | +1.43 pp | 0.5811 | +1.54 pp | 174.3 | -18.8 |
+
+## Artifact checklist
+
+- Local packed-eval snapshots: `per_iter_results/{vanilla,retok,td}_iter{585,715,834}.json`.
+- Local BPC snapshots: `per_iter_results/intrinsic/*_iter{585,715,834}_fair.json`.
+- Local new-token diagnostics: `per_iter_results/diagnostics/{retok,td}_iter{585,715,834}_new_token_diagnostics.json`.
+- Regenerated plots are written to `plots/`.
+- Final remote packed eval job: `2376082`, state `COMPLETED`, exit `0:0`, elapsed `00:59:51`.

results/benchmark_table.csv

@@ -0,0 +1,5 @@
+arm,greek_aggregate,greek_delta_vs_iter476_pp,english_retention,english_delta_vs_iter476_pp,multilingual,multilingual_delta_vs_iter476_pp,bpc_lower_better,bpc_delta
+Vanilla,0.4339,-0.70,0.6782,-0.36,0.4923,0.22,0.4724,-0.0182
+ReTok,0.4246,0.96,0.6786,0.36,0.4864,-0.09,0.5390,-0.0349
+TD layer11,0.4344,0.90,0.6865,0.38,0.4967,0.68,0.5054,-0.0256
+

results/continuation_3p5b_summary.json

@@ -0,0 +1,364 @@
+{
+  "diag_rows": [
+    {
+      "arm": "retok",
+      "mean_rank": 228.9052154654546,
+      "mean_rank_delta_from_585": -31.404831868911288,
+      "top1": 0.3798836182563004,
+      "top1_delta_from_585": 0.019972352098200752,
+      "top5": 0.5469290563525036,
+      "top5_delta_from_585": 0.021119522892752207
+    },
+    {
+      "arm": "td",
+      "mean_rank": 174.2876756944545,
+      "mean_rank_delta_from_585": -18.7823704818841,
+      "top1": 0.41053515336624063,
+      "top1_delta_from_585": 0.014301637161096115,
+      "top5": 0.5811415254114616,
+      "top5_delta_from_585": 0.015358337072796546
+    }
+  ],
+  "generated_utc": "2026-05-25T05:23:53+00:00",
+  "group_rows": [
+    {
+      "EN retention": {
+        "delta": -0.0036023994242395663,
+        "iter476": 0.6817902499718899,
+        "iter834": 0.6781878505476503
+      },
+      "Greek": {
+        "delta": -0.0069518253078383885,
+        "iter476": 0.44085471366912354,
+        "iter834": 0.43390288836128516
+      },
+      "Multilingual": {
+        "delta": 0.002230923643585403,
+        "iter476": 0.4900917349227056,
+        "iter834": 0.492322658566291
+      },
+      "arm": "vanilla",
+      "bpc": {
+        "delta": -0.018193672749048873,
+        "iter476": 0.4905790115645303,
+        "iter834": 0.47238533881548145
+      }
+    },
+    {
+      "EN retention": {
+        "delta": 0.0036478269859219203,
+        "iter476": 0.6749804337638333,
+        "iter834": 0.6786282607497552
+      },
+      "Greek": {
+        "delta": 0.00956345274685877,
+        "iter476": 0.41502996652924373,
+        "iter834": 0.4245934192761025
+      },
+      "Multilingual": {
+        "delta": -0.0009003749821289708,
+        "iter476": 0.48733830931505984,
+        "iter834": 0.48643793433293087
+      },
+      "arm": "retok",
+      "bpc": {
+        "delta": -0.03491945208518521,
+        "iter476": 0.57388731113822,
+        "iter834": 0.5389678590530348
+      }
+    },
+    {
+      "EN retention": {
+        "delta": 0.0037544227559322962,
+        "iter476": 0.6827725274715655,
+        "iter834": 0.6865269502274978
+      },
+      "Greek": {
+        "delta": 0.008970722628639627,
+        "iter476": 0.4253979832454272,
+        "iter834": 0.43436870587406684
+      },
+      "Multilingual": {
+        "delta": 0.006784718226652775,
+        "iter476": 0.48993351359709303,
+        "iter834": 0.4967182318237458
+      },
+      "arm": "td",
+      "bpc": {
+        "delta": -0.02564778560419223,
+        "iter476": 0.531084014463596,
+        "iter834": 0.5054362288594038
+      }
+    }
+  ],
+  "iterations": [
+    476,
+    585,
+    715,
+    834
+  ],
+  "task_rows": [
+    {
+      "deltas": {
+        "retok": 0.002350092579404661,
+        "td": 0.005554764278592805,
+        "vanilla": -0.0009257940464321157
+      },
+      "group": "EN retention",
+      "label": "MMLU",
+      "task": "mmlu",
+      "values": {
+        "retok": 0.5565446517590087,
+        "td": 0.5556188577125766,
+        "vanilla": 0.5330437259649623
+      },
+      "winner": "retok"
+    },
+    {
+      "deltas": {
+        "retok": 0.006871141206930886,
+        "td": 0.0026887074287991197,
+        "vanilla": -0.0020912168890659943
+      },
+      "group": "EN retention",
+      "label": "HellaSwag",
+      "task": "hellaswag",
+      "values": {
+        "retok": 0.7556263692491536,
+        "td": 0.7631945827524398,
+        "vanilla": 0.7573192591117307
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": -0.005892255892255949,
+        "td": 0.002946127946127919,
+        "vanilla": 0.0012626262626262985
+      },
+      "group": "EN retention",
+      "label": "ARC Easy",
+      "task": "arc_easy",
+      "values": {
+        "retok": 0.7575757575757576,
+        "td": 0.7836700336700336,
+        "vanilla": 0.7866161616161617
+      },
+      "winner": "vanilla"
+    },
+    {
+      "deltas": {
+        "retok": 0.009385665529010212,
+        "td": 0.008532423208191142,
+        "vanilla": -0.004266211604095571
+      },
+      "group": "EN retention",
+      "label": "ARC Challenge",
+      "task": "arc_challenge",
+      "values": {
+        "retok": 0.5247440273037542,
+        "td": 0.5358361774744027,
+        "vanilla": 0.523037542662116
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": -0.0010881392818281599,
+        "td": -0.002720348204570233,
+        "vanilla": -0.0021762785636560977
+      },
+      "group": "EN retention",
+      "label": "PIQA",
+      "task": "piqa",
+      "values": {
+        "retok": 0.7850924918389554,
+        "td": 0.7894450489662677,
+        "vanilla": 0.7927094668117519
+      },
+      "winner": "vanilla"
+    },
+    {
+      "deltas": {
+        "retok": 0.010260457774269871,
+        "td": 0.005524861878453136,
+        "vanilla": -0.013417521704814472
+      },
+      "group": "EN retention",
+      "label": "Winogrande",
+      "task": "winogrande",
+      "values": {
+        "retok": 0.6921862667719021,
+        "td": 0.691397000789266,
+        "vanilla": 0.6764009471191792
+      },
+      "winner": "retok"
+    },
+    {
+      "deltas": {
+        "retok": 0.001391122832608005,
+        "td": 0.013387865279933964,
+        "vanilla": -0.006411768438843168
+      },
+      "group": "Multilingual",
+      "label": "Global MMLU",
+      "task": "global_mmlu",
+      "values": {
+        "retok": 0.4602990774298503,
+        "td": 0.4628181979660652,
+        "vanilla": 0.4504750950855104
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": -0.007090909090909148,
+        "td": 0.0041818181818181754,
+        "vanilla": 0.010909090909090868
+      },
+      "group": "Multilingual",
+      "label": "XCOPA",
+      "task": "xcopa",
+      "values": {
+        "retok": 0.592,
+        "td": 0.6156363636363636,
+        "vanilla": 0.6178181818181818
+      },
+      "winner": "vanilla"
+    },
+    {
+      "deltas": {
+        "retok": 0.0029986613119142858,
+        "td": 0.0027844712182061304,
+        "vanilla": 0.0021954484605086755
+      },
+      "group": "Multilingual",
+      "label": "XNLI",
+      "task": "xnli",
+      "values": {
+        "retok": 0.4070147255689424,
+        "td": 0.41170013386880855,
+        "vanilla": 0.4086746987951807
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": 0.016165788349237986,
+        "td": 0.017732516735507742,
+        "vanilla": -0.006907847884916718
+      },
+      "group": "Greek",
+      "label": "Greek MMLU",
+      "task": "global_mmlu_full_el",
+      "values": {
+        "retok": 0.4152542372881356,
+        "td": 0.4036462042444096,
+        "vanilla": 0.4144708730950007
+      },
+      "winner": "retok"
+    },
+    {
+      "deltas": {
+        "retok": 0.03260869565217395,
+        "td": -0.0036231884057971175,
+        "vanilla": -0.016304347826086973
+      },
+      "group": "Greek",
+      "label": "INCLUDE-44 Greek",
+      "task": "include_base_44_greek_few_shot_en",
+      "values": {
+        "retok": 0.4057971014492754,
+        "td": 0.4003623188405797,
+        "vanilla": 0.40217391304347827
+      },
+      "winner": "retok"
+    },
+    {
+      "deltas": {
+        "retok": -0.0033333333333332993,
+        "td": 0.011111111111111072,
+        "vanilla": -0.005555555555555536
+      },
+      "group": "Greek",
+      "label": "Belebele Greek",
+      "task": "belebele_ell_Grek",
+      "values": {
+        "retok": 0.49333333333333335,
+        "td": 0.5388888888888889,
+        "vanilla": 0.5077777777777778
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": 0.017918088737201354,
+        "td": 0.0025597269624573205,
+        "vanilla": -0.007679180887372017
+      },
+      "group": "Greek",
+      "label": "ARC Challenge MT-el",
+      "task": "arc_challenge_mt_el",
+      "values": {
+        "retok": 0.38993174061433444,
+        "td": 0.4061433447098976,
+        "vanilla": 0.4129692832764505
+      },
+      "winner": "vanilla"
+    },
+    {
+      "deltas": {
+        "retok": -0.005622489959839383,
+        "td": 0.004819277108433773,
+        "vanilla": -0.01887550200803212
+      },
+      "group": "Greek",
+      "label": "XNLI Greek",
+      "task": "xnli_el",
+      "values": {
+        "retok": 0.36947791164658633,
+        "td": 0.385140562248996,
+        "vanilla": 0.38313253012048193
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": -0.0007925802174291174,
+        "td": 0.010195614888764415,
+        "vanilla": -0.02334034299290566
+      },
+      "group": "Greek",
+      "label": "XQuAD Greek F1",
+      "task": "xquad_el",
+      "values": {
+        "retok": 0.3083596106010522,
+        "td": 0.3363996221856959,
+        "vanilla": 0.2867958412158067
+      },
+      "winner": "td"
+    },
+    {
+      "deltas": {
+        "retok": 0.010000000000000009,
+        "td": 0.019999999999999907,
+        "vanilla": 0.030000000000000027
+      },
+      "group": "Greek",
+      "label": "PIQA Greek",
+      "task": "global_piqa_completions_ell_grek",
+      "values": {
+        "retok": 0.59,
+        "td": 0.57,
+        "vanilla": 0.63
+      },
+      "winner": "vanilla"
+    }
+  ],
+  "tokens_b": {
+    "476": 1.996488704,
+    "585": 2.45366784,
+    "715": 2.99892736,
+    "834": 3.498049536
+  }
+}

tokenizer/modern-greek-17408/README.md

@@ -0,0 +1,22 @@
+# Modern Greek Tokenizer, 148480 Vocab
+
+This is the canonical modern Greek tokenizer extension for Apertus.
+
+Properties:
+
+- base model: `swiss-ai/Apertus-8B-2509`;
+- method: merge-rule BPE extension, not `add_tokens`;
+- base vocab preserved: `131072`;
+- added modern Greek tokens: `17408`;
+- total vocab: `148480`;
+- alignment: divisible by `128` and `256`;
+- first 1000 special/reserved ids preserved;
+- `tokenizer.json` SHA-256:
+  `358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394`.
+
+Selection evidence is in:
+
+```text
+../../provenance/tokenizer-selection/CHOSEN_CUTOFF.md
+```
+

tokenizer/modern-greek-17408/manifest.json

@@ -0,0 +1,33 @@
+{
+  "name": "apertus_greek_modern_only_148480",
+  "purpose": "Apertus-compatible modern-Greek-only tokenizer: base 131,072 + C3 modern Greek (+17,408 curated+backfilled) = 148,480 = 256 × 580. Use this for the three-arm Vanilla/ReTok/Distillation init comparison.",
+  "vocab_size": 148480,
+  "base_apertus_vocab": 131072,
+  "added_modern_greek_c3": 17408,
+  "added_polytonic_greek": 0,
+  "alignment": {
+    "128": true,
+    "256": true,
+    "factor_256": 580
+  },
+  "first_1000_preserved": true,
+  "front_end_contract": {
+    "normalizer": null,
+    "pre_tokenizer": "Sequence(Split(GPT-2 regex) → ByteLevel)",
+    "decoder": "ByteLevel",
+    "model_type": "BPE",
+    "special_tokens": {
+      "unk": 0,
+      "bos": 1,
+      "eos": 2,
+      "pad": 3,
+      "reserved_count": 1000
+    }
+  },
+  "loadable_via": "transformers.AutoTokenizer.from_pretrained() → PreTrainedTokenizerFast",
+  "sha256": {
+    "tokenizer.json": "358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394",
+    "tokenizer_config.json": "ea64a17b41e1deaa7469212f413676129f33977ca3a48767f0ca68dc346df502",
+    "special_tokens_map.json": "816ec96e37c6d15e3cbc535dc146c898a7218f209fc154384f31fc1e6ad31ba5"
+  }
+}

tokenizer/modern-greek-17408/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}