diff --git a/.gitattributes b/.gitattributes index d610e72641b9ae63e30d54739cad538726009b8d..1cc3aefcc0b37fa3599d108971fb3134bf74de06 100644 --- a/.gitattributes +++ b/.gitattributes @@ -39,3 +39,12 @@ subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_ex subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_per_task_van_td.png filter=lfs diff=lfs merge=lfs -text subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_van_td.png filter=lfs diff=lfs merge=lfs -text subprojects/03_apertus_extension_and_embedding_adaptation/03_4_implementation_experiments/init_bakeoff/eval/trajectory_analysis_20260524/plots/trajectories_van_td_gap.png filter=lfs diff=lfs merge=lfs -text +results/plots/global_mmlu_full_el_subcategories_van_td.png filter=lfs diff=lfs merge=lfs -text +results/plots/include_base_44_greek_subjects_van_td.png filter=lfs diff=lfs merge=lfs -text +results/plots/intrinsic_trajectories.png filter=lfs diff=lfs merge=lfs -text +results/plots/intrinsic_van_td.png filter=lfs diff=lfs merge=lfs -text +results/plots/trajectories.png filter=lfs diff=lfs merge=lfs -text +results/plots/trajectories_per_task.png filter=lfs diff=lfs merge=lfs -text +results/plots/trajectories_per_task_van_td.png filter=lfs diff=lfs merge=lfs -text +results/plots/trajectories_van_td.png filter=lfs diff=lfs merge=lfs -text +results/plots/trajectories_van_td_gap.png filter=lfs diff=lfs merge=lfs -text diff --git a/ARTIFACT_GRAPH.md b/ARTIFACT_GRAPH.md new file mode 100644 index 0000000000000000000000000000000000000000..085d912f2f5c7cae7749be371d52ff2c969f5459 --- /dev/null +++ b/ARTIFACT_GRAPH.md @@ -0,0 +1,51 @@ +# Artifact Graph + +```text +swiss-ai/Apertus-8B-2509 + | + | continuous BPE on GlossAPI + HPLT 50/50 + v +C3 full candidate + base 131072 + 25600 = 156672 + | + | cutoff sweep, curation, backfill, alignment + v +tokenizer/modern-greek-17408 + base 131072 + 17408 = 148480 + | + | optional polytonic stack + v +tokenizer/polytonic-plus-5120 + base 131072 + modern 17408 + polytonic 5120 = 153600 + +fffoivos/glossapi-greek-nanochat-pretraining-dataset + + nanochat internal dedup metadata + + Apertus overlap drop overlay + | + v +selected_after_apertus_and_internal_dedup.parquet + | + | 70/24/4/2 Greek/replay/code/math mix + v +bulk_mix.nfc.jsonl + | + | Megatron indexed dataset preprocessing + v +bulk_mix_base_nfc_megatron/bulk_mix_text_document.{bin,idx} + +modern tokenizer + CPT mix + | + | Vanilla, ReTok, Centroid, Token Distillation layer 11 + v +2B bakeoff and 3.5B continuation + | + v +checkpoints/td-layer11-cpt-3p5b-iter834 + | + v +results/ +``` + +The large checkpoint and dataset payloads are represented by hydration pointers +unless explicitly uploaded as model weights. + diff --git a/MANIFEST.json b/MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..c7356225d8e17e28e2e264d12eaa5b9f5dd0fe2e --- /dev/null +++ b/MANIFEST.json @@ -0,0 +1,150 @@ +{ + "release_name": "apertus-tokenizer-extension", + "generated_date": "2026-05-25", + "source_repo": "https://github.com/fffoivos/glossapi-tokenizer-extension", + "source_subproject": "subprojects/03_apertus_extension_and_embedding_adaptation", + "source_commit_for_code_refs": "7c67cf11eb3d31f0ddd006f7d709709037db93a1", + "artifacts": [ + { + "artifact_id": "tokenizer.modern-greek-17408", + "type": "tokenizer", + "status": "payload-present", + "public_path": "tokenizer/modern-greek-17408", + "base_model": "swiss-ai/Apertus-8B-2509", + "vocab_size": 148480, + "base_vocab_size": 131072, + "added_modern_greek_tokens": 17408, + "sha256_tokenizer_json": "358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394", + "review_docs": [ + "provenance/tokenizer-selection/CHOSEN_CUTOFF.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "tokenizer.polytonic-plus-5120", + "type": "tokenizer", + "status": "payload-present", + "public_path": "tokenizer/polytonic-plus-5120", + "base_model": "swiss-ai/Apertus-8B-2509", + "vocab_size": 153600, + "base_vocab_size": 131072, + "added_modern_greek_tokens": 17408, + "added_polytonic_greek_tokens": 5120, + "sha256_tokenizer_json": "b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad", + "review_docs": [ + "provenance/tokenizer-selection/CHOSEN_CUTOFF.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "dataset.cpt-7b-mix", + "type": "dataset", + "status": "hydration-pointer", + "public_path": "training-data/cpt-7b-mix", + "clariden_jsonl": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix.nfc.jsonl", + "clariden_megatron_prefix": "/iopsstor/scratch/cscs/fffoivos/cpt_corpus/bulk_mix_base_nfc_megatron/bulk_mix_text_document", + "rows": 5754172, + "base_tokenized_tokens": 9831704774, + "mix_builder_target_tokens": 7000141612, + "review_docs": [ + "provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md", + "provenance/dataset-build/MIX_RECIPE.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "checkpoint.td-layer11-init-r17-tp2", + "type": "checkpoint", + "status": "hydration-pointer", + "public_path": "checkpoints/td-layer11-init-r17-tp2", + "format": "Megatron torch_dist TP=2, R17 patched", + "clariden_path": "/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release", + "depends_on": [ + "tokenizer.modern-greek-17408" + ], + "review_docs": [ + "provenance/token-distillation/RUN_LOG_20260523.md", + "provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "checkpoint.td-layer11-cpt-3p5b-iter834", + "type": "checkpoint", + "status": "hydration-pointer", + "public_path": "checkpoints/td-layer11-cpt-3p5b-iter834", + "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion", + "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834", + "clariden_hf_eval_copy": "/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf", + "depends_on": [ + "tokenizer.modern-greek-17408", + "dataset.cpt-7b-mix", + "checkpoint.td-layer11-init-r17-tp2" + ], + "review_docs": [ + "results/SUMMARY.md", + "provenance/evals/CONTINUATION_3P5B_RESULTS_20260525.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "checkpoint.baseline-vanilla-3p5b-iter834", + "type": "checkpoint", + "status": "hydration-pointer", + "public_path": "checkpoints/baselines/vanilla-3p5b-iter834", + "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion", + "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834", + "depends_on": [ + "dataset.cpt-7b-mix" + ], + "review_docs": [ + "results/SUMMARY.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "checkpoint.baseline-retok-3p5b-iter834", + "type": "checkpoint", + "status": "hydration-pointer", + "public_path": "checkpoints/baselines/retok-3p5b-iter834", + "format": "Megatron torch_dist TP=2 checkpoint plus HF eval conversion", + "clariden_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834", + "depends_on": [ + "tokenizer.modern-greek-17408", + "dataset.cpt-7b-mix" + ], + "review_docs": [ + "results/SUMMARY.md" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + }, + { + "artifact_id": "results.continuation-3p5b", + "type": "evaluation-summary", + "status": "compact-evidence", + "public_path": "results", + "review_docs": [ + "results/SUMMARY.md", + "results/benchmark_table.csv" + ], + "github_code_refs": [ + "code-links/github_subproject_manifest.json" + ] + } + ] +} + diff --git a/README.md b/README.md index f2847c2119a62b48c8c12d5df41336a3bb69c53d..9f72e8c21c1256c2fbbe79737358deaf22005fb6 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,99 @@ ---- -language: -- el -tags: -- tokenizer -- bpe -- greek -- apertus -- tokenizer-extension -- byte-level-bpe -datasets: -- fffoivos/glossapi-greek-nanochat-pretraining-dataset -base_model: -- swiss-ai/Apertus-8B-2509 ---- - -# Apertus Tokenizer Extension - -Current canonical artifact: **C3 + 17,408 curated/backfilled added units**. - -The active ship tokenizer is: +# Apertus Greek Tokenizer Extension + +This release packages the Greek tokenizer-extension work around its main +artifacts: + +1. a modern Greek merge-rule tokenizer extension for `swiss-ai/Apertus-8B-2509`; +2. an optional polytonic/ancient Greek stacked tokenizer; +3. the CPT dataset recipe and hydration pointers used for the bakeoff; +4. the init and Token Distillation checkpoint pointers; +5. compact benchmark evidence from the 2B bakeoff and 3.5B continuation. + +The runnable scripts live in GitHub, not in this Hugging Face artifact repo: ```text -tokenizers/c3_added_17408_curated_padded/ +https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation ``` -It starts from `swiss-ai/Apertus-8B-2509`, preserves the Apertus base ids -`0..131071` verbatim, adds `17,408` C3 Greek-extension BPE units, and keeps -the final vocab aligned at `148,480 = 128 * 1160 = 256 * 580`. +## Main Artifacts -## Canonical Tokenizer +| Artifact | Status | Path | +|---|---|---| +| Modern Greek tokenizer, 148,480 vocab | payload included | `tokenizer/modern-greek-17408/` | +| Polytonic-stacked tokenizer, 153,600 vocab | payload included | `tokenizer/polytonic-plus-5120/` | +| TD layer 11 CPT checkpoint at 3.5B | hydration pointer | `checkpoints/td-layer11-cpt-3p5b-iter834/` | +| TD layer 11 R17-patched init | hydration pointer | `checkpoints/td-layer11-init-r17-tp2/` | +| Vanilla/ReTok 3.5B baselines | hydration pointers | `checkpoints/baselines/` | +| CPT 7B text mix recipe | manifest and pointer | `training-data/cpt-7b-mix/` | +| 3.5B benchmark summary | compact evidence | `results/` | +| Source-code links | GitHub pointers | `code-links/` | +| Audit trail | compact provenance | `provenance/` | -| Path | Vocab | Added units | Curation | SHA-256 tokenizer.json | -| --- | ---: | ---: | --- | --- | -| `tokenizers/c3_added_17408_curated_padded/` | 148,480 | 17,408 | 69 noisy raw-cutoff tokens structurally skipped and backfilled | `358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394` | +## Tokenizers -The cutoff can be revisited after downstream embedding adaptation and CPT, -but this is the current handoff artifact for the Apertus Greek tokenizer -extension line. +Modern-only tokenizer: -## Cutoff Evidence +- base Apertus vocab: `131072`; +- added modern Greek C3 tokens: `17408`; +- total vocab: `148480`; +- SHA-256 for `tokenizer.json`: + `358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394`. -The compact evidence bundle is under: +Polytonic-stacked tokenizer: -```text -experiments/02_1_7_intrinsic_eval_sweep_20260518/ -``` +- base Apertus vocab: `131072`; +- modern Greek extension: `17408`; +- polytonic/ancient Greek extension: `5120`; +- total vocab: `153600`; +- SHA-256 for `tokenizer.json`: + `b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad`. -Important files: +Both tokenizers preserve Apertus base ids and the first 1000 special/reserved +ids. -- `CHOSEN_CUTOFF.md` — pinned decision and downstream contract. -- `REPORT.md` — intrinsic/fertility/MorphScore evidence base. -- `artifacts/plots/` — report plots for the cutoff decision. -- `manifests/curated_padded_at_17408_manifest.json` — exact backfilled construction. -- `manifests/removal_mask_at_17408.jsonl` — the 69 in-cutoff tokens filtered out. -- `firing_counts_c3_added_17408_curated_padded/` — compact token firing-count attribution artifacts for the exact C3 BPE training corpus, split by GlossAPI-nanochat, HPLT, combined C3, and exact `source_dataset`. +## Dataset -The full sweep zoo is intentionally not mirrored here. This repo keeps the -canonical tokenizer and just enough evidence to explain why it is current. -Raw local variants, TokEval vendor caches, parquet outputs, and large geometry -arrays remain outside the Hub artifact set. +The CPT text mix is built from: -## Historical Artifacts +- `fffoivos/glossapi-greek-nanochat-pretraining-dataset`; +- nanochat internal dedup metadata; +- Apertus-overlap drop overlay from + `fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z`; +- non-Greek replay, code, and math. -Older C1/C2/C3 full-tokenizer candidates and language-attribution artifacts are -still present in their historical paths, including: +Bulk recipe: -- `continuous/` -- `fresh/` -- `analysis/` -- `artifacts/language_attribution_20260515/` +- Greek: `70%`; +- non-Greek replay: `24%`; +- code: `4%`; +- math: `2%`. -Those are retained for provenance, but the current tokenizer handoff is the -17,408 curated/backfilled artifact above. +The final text stream is about `7.0B` extended-tokenizer-budget tokens. The +same NFC JSONL stream contains `9,831,704,774` base-tokenized Megatron tokens +after preprocessing. -## Source Code +## Current Result Anchor -Process docs and scripts live in GitHub: +The latest compact result is the 3.5B continuation summary in `results/`. -```text -https://github.com/fffoivos/glossapi-tokenizer-extension -``` +At iter 834: -Relevant commit for this upload: +| Arm | Greek aggregate | English retention | Multilingual | Heldout BPC, lower better | +|---|---:|---:|---:|---:| +| Vanilla | 0.4339 | 0.6782 | 0.4923 | 0.4724 | +| ReTok | 0.4246 | 0.6786 | 0.4864 | 0.5390 | +| TD layer 11 | 0.4344 | 0.6865 | 0.4967 | 0.5054 | + +Reading: TD layer 11 is the strongest final benchmark arm overall after the +3.5B continuation. Vanilla remains the heldout-BPC reference. + +## Source And Provenance + +- `MANIFEST.json`: machine-readable artifact inventory. +- `ARTIFACT_GRAPH.md`: dependency graph. +- `code-links/github_subproject_manifest.json`: source-code pointers. +- `provenance/`: compact documents and verification outputs for tokenizer + selection, dataset build, TD, conversion, and eval. +- `archive/legacy-hf-layout-index.md`: index of older HF folders retained for + traceability. -```text -9a6b039 Add firing count attribution workflow -``` diff --git a/SHA256SUMS b/SHA256SUMS index 041523c95c6f769c1eb05700414732104dcbb62f..5e54f3c24528123f5a94fb2c583b8cec28f573e0 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,23 +1,59 @@ -e6e321ad419e32ce43427bba05a02a23133f05500a3486f98c5de232830c1844 ./README.md -5c2140f0d646a2f5eaf7ea07c033fade7e28b37468eeb8595f61f6044f6cf109 ./SHA256SUMS -f6742a090b39f1f576d18481bb1e284720275b0ab417393f219650821e83a72a ./experiments/02_1_7_intrinsic_eval_sweep_20260518/CHOSEN_CUTOFF.md -3d68bbd6e6c9e25df597f46819cbe57e336b62b8a9b5b441aadeae5346e70eb7 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/README.md -9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/REPORT.md -60a57f77e13d8204c7728e6e5a117cb947c156c97e2c57db7de587965ada58bf ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/morphscore_greek_results.json -7dd2ae959af9504d522cf4b606c573e6cefdd71c009fe8a7269eb051382351f0 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/curated_metrics_by_slice.csv -76ad56410ada9ef2d7649d3babb757287993982e03fc98057e7289ec837c990c ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/extended_metrics_by_slice.csv -190c58ce62f8878c92499ef7da9bceb26f27a1b5f25927e6681d80df59359c79 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/metrics_by_slice.csv -f823dcc20f7cc2d7435fda6c5c09e65d704de10d37330f9ed930fdcf7f75506b ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/padded_17408_metrics.csv -9e94927189811f9c5df2e6f75c3e4ffc4738d4305a35eef2b8499c6bf15c5acf ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/summary.json -7bbfd6c76cc88190a2334ab45af03f77e8389830e46e8dcc4ef896f7e2e3b8fc ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/extended_4metric_to_25k.png -0f472db11e1f6809b2369727de9d78e224d1c5916701141eae0c5207149d0681 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/fertility_per_in_domain_slice.png -7b90ed2c76344942c48fbb445386e16fde9e0a9af0fdbd03e96054aee1ec38d3 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/headline_4metric_in_domain.png -f72541da46b7539acca2c0fc9d641d751f0461528598e9d82651e3722681c226 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/knee_analysis.png -4f0148ff0c94a8b67e329517ab260963daa027c0ab64ac992cffe4e95c073665 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/morphscore_greek.png -763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/curated_padded_at_17408_manifest.json -7874469554d319addc7f113e6b5d4b7f8984e3bd4db9b3dcaa3e4e476cf392f0 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/removal_mask_at_17408.jsonl -0b3c2f295d1f14c9409b39ca2e6fc98346037a7d11a7cd931b59fcff7824b2e9 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/tokeval_commit.txt -7147abc0dd32b702a399e02f47cea5576594ee6ec0d089a4e1fb6d822797aad3 ./tokenizers/c3_added_17408_curated_padded/README.md -358c249e2fb29060c6b73157d428853b0c48710deffc8ee670ab1013880946c9 ./tokenizers/c3_added_17408_curated_padded/special_tokens_map.json -358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394 ./tokenizers/c3_added_17408_curated_padded/tokenizer.json -737b93393573376f6b4707ebfbb17688d0c358cba0daa3be12b4f9f57f9cb2ae ./tokenizers/c3_added_17408_curated_padded/tokenizer_config.json +d528c230d2d3e151776332036669794eaacc5962a1dd3b3805a64c56e2f539ff ./ARTIFACT_GRAPH.md +6c8fe007c045e4be44f2825d16724be5cc80d53956b1aba3d64b8843f42f2c9c ./MANIFEST.json +e055076dd9f8fc307d2056f5c46c9f6cd99463dfe610758a45796e98200bf3cc ./README.md +dc2f59def39f4d46b645608e789ebca000b3f3257336c393b234da9b9c26f607 ./archive/legacy-hf-layout-index.md +620d1591179643c31aca8c3ba98492204fc2dd3388afff5764726d632b28fd09 ./archive/legacy_SHA256SUMS_20260525 +2eee70665c1fd320ebc5d664d5fe3b526be32adf33be71f63a0edd3b52545076 ./checkpoints/baselines/retok-3p5b-iter834/README.md +be50de74fda67fb79a2d4ae616f3eaa0800e46b641b1ae3bd5a5d12e1fd6aeb2 ./checkpoints/baselines/retok-3p5b-iter834/hydration.md +1a4aff5d2d4dec2424a50fb7921d3bb92fc9a92d3e6b539a4bc2b542dfefa9a5 ./checkpoints/baselines/retok-3p5b-iter834/manifest.json +ea3e33007aa0ddfff2f88039b9f7d6a16c1317b14bee999ab762fc6867000649 ./checkpoints/baselines/vanilla-3p5b-iter834/README.md +1531171806fc72e45ba4467a1872a40ed1360ae5b6926cd7e313f818d924e4cd ./checkpoints/baselines/vanilla-3p5b-iter834/hydration.md +c7203eaed58b9b9a23d21e7b25ac37dbf9ee07538707626181f06795b1ae26c1 ./checkpoints/baselines/vanilla-3p5b-iter834/manifest.json +553be4bd24ccfc8bef210913c5e6ff7a26ca909b782bbd266ad927db07709581 ./checkpoints/td-layer11-cpt-3p5b-iter834/README.md +6d0724888604be70abae50448eb120160710b875b42dec21ce7419a82fa60c01 ./checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md +69b48c1c95a4c79cfdbab0ec12a669e0bc74549294e3c38dfde57d547dd2d2f6 ./checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json +4bcb2a3a6316ee1373a170d0d4bb9e32ec25dafb942c10fb781188216faceafd ./checkpoints/td-layer11-init-r17-tp2/README.md +da19cd1247b0401eb6d6e25d8c0fc496ee5d93b2e745b05ce8ff59b6b9fd8c57 ./checkpoints/td-layer11-init-r17-tp2/hydration.md +98d6a3fa9b342949b4972514925bff2952de06a202bcfcc56b449741ea4530c6 ./checkpoints/td-layer11-init-r17-tp2/manifest.json +44a71b0a1730bf13f3387ccf15967bf036f519f002b775cd4e7a671a44c9830a ./code-links/README.md +44705bc53f80deba2ac02336a29a77a6fa1b83431ec10fe9b100b5493903956f ./code-links/github_subproject_manifest.json +06ddc69f4787f470b85d4be55992f898b98d8ab4fd8ab34a89a4d3bb96eb7e33 ./provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json +496fa7d6f5622d6038aa2e333adbbd142ac83341e834acedc697892d66031c44 ./provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json +61c13b8d63a692fad24e7cdf5fe5b5d6e2fb82bc4cf61981ea7c8b62b42569a6 ./provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md +6ab3047ac06c925073ca06c1a91e91bf1c1ce4e4a1336f41f6ef35769228ebd1 ./provenance/dataset-build/MIX_RECIPE.md +2dafea9a789b717e1f8b6fb931d8b47483e3b8433ceb1fe4b8064afa3bf5489a ./provenance/dataset-build/bulk_mix_text_document.manifest.json +8fbed782f3250003ef427d56ff27ae4379311da5a6263e078466ce06d6078bd2 ./provenance/evals/CONTINUATION_3P5B_RESULTS_20260525.md +d336b0f9fa29db6ce9052ae5fcac15e55acf8f4dc0e1385c820076c43c1e4a42 ./provenance/evals/EVAL_RECIPE.md +c10f3693691c657945530ee70bfcca8ad85bd3b42939f1ff321c9aa27b085112 ./provenance/evals/V4_BENCHMARK_COMPARISON.md +80e2fda4e1c0ce1e2e066986add6c9e76db592f9d9b20b0b0f6857209c101d7d ./provenance/token-distillation/RUN_LOG_20260523.md +acda2215c1443fdd3ad25ced78085c121139640adf6d4f3037f776ec6e13f47d ./provenance/token-distillation/TOKEN_DISTILLATION_PLAN.md +fada1d3fa64b8060b17a65bc8b4fa8739e560454fcfe172d681ce055a51a38ba ./provenance/tokenizer-selection/CHOSEN_CUTOFF.md +9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25 ./provenance/tokenizer-selection/CUTOFF_SWEEP_REPORT.md +763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895 ./provenance/tokenizer-selection/curated_padded_at_17408_manifest.json +6046eb8f4cabda22fddf8cf5ed1581267b854221d03253e2994d304572b4f320 ./provenance/tokenizer-selection/firing_count_20260518_run_summary_augmented.json +8fbed782f3250003ef427d56ff27ae4379311da5a6263e078466ce06d6078bd2 ./results/SUMMARY.md +712a3c96216fe3159433b9b9d886a8cb22b51001a35a3778a7e5d6ba99909772 ./results/benchmark_table.csv +669dc4a1c3252f994495d2fb1459145222443bcfa6f6ca1221e8ee92b602c56b ./results/continuation_3p5b_summary.json +13337a7976d4ca46c6c3db03014a2207ba81a42796c285410dc3f4ad26007cd9 ./results/plots/global_mmlu_full_el_subcategories_van_td.png +6375661e9c50fd4b9f32db79a9c77375aaadbb878fab8a61019802978af3ac0e ./results/plots/include_base_44_greek_subjects_van_td.png +a44571c0aca767ace369a89534632a9b4618bc01391c0b20bb0f04f1c34a8830 ./results/plots/intrinsic_trajectories.png +975e0c58604e6189f45182e07cf178bb94bead39130385aae36ed95c065ed2ad ./results/plots/intrinsic_van_td.png +ab072600c16be4aab4fe36ebd5b7fe3a216f1bf5ac391687733667f6231ff486 ./results/plots/trajectories.png +375efc4b55341c0a1060fb9d7ccaa8774db66c8ca554683100449e15b1541a44 ./results/plots/trajectories_per_task.png +ea2a02ecd33210ada4c9a5d1f7832246de07205cc4bbc5679aedb6959f93bfbe ./results/plots/trajectories_per_task_van_td.png +a54f94603549ddaf022297a2f691a688fb6283d793394f482531211b42dfa3a3 ./results/plots/trajectories_van_td.png +d49ee1c80666418e6f36b649b63c9a80fb2808bb365d31e481a6c58c97053d11 ./results/plots/trajectories_van_td_gap.png +55ce65390b6fbdc16a81f22e3d78c3a452e72192011b72289c0180a656052273 ./tokenizer/modern-greek-17408/README.md +09c84b4086e9f800174942dcba688d8eefa8e40f3b60c20341699a6a7a11e2bf ./tokenizer/modern-greek-17408/manifest.json +816ec96e37c6d15e3cbc535dc146c898a7218f209fc154384f31fc1e6ad31ba5 ./tokenizer/modern-greek-17408/special_tokens_map.json +358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394 ./tokenizer/modern-greek-17408/tokenizer.json +ea64a17b41e1deaa7469212f413676129f33977ca3a48767f0ca68dc346df502 ./tokenizer/modern-greek-17408/tokenizer_config.json +208ae8ea5f95c9182dc669c150aaf0c720e5937a402c72062fd83e1960978d6a ./tokenizer/polytonic-plus-5120/README.md +3881e1940ca013e9ec040fac3fb3a08a8f9b871e5105a0fec8165e0c6759acae ./tokenizer/polytonic-plus-5120/manifest.json +816ec96e37c6d15e3cbc535dc146c898a7218f209fc154384f31fc1e6ad31ba5 ./tokenizer/polytonic-plus-5120/special_tokens_map.json +b1eeb739a564b3abd33c1b85a16162b8284d98f9ab5d67528d3cbe8a82e9cbad ./tokenizer/polytonic-plus-5120/tokenizer.json +ea64a17b41e1deaa7469212f413676129f33977ca3a48767f0ca68dc346df502 ./tokenizer/polytonic-plus-5120/tokenizer_config.json +d2f04aa396cf6558502b919b4503d248aef0ccfe08251625d75f8a9314272787 ./training-data/cpt-7b-mix/README.md +f16d35ba906d5439709f189b1dc1253ba6e3d8518a828aac71c04fd42fefd0cf ./training-data/cpt-7b-mix/hydration.md +cafd9e0b15ab1d9f4f0eb7bc2939ccab29e9b6ab86ccc1f20e9a42a86b5760c0 ./training-data/cpt-7b-mix/manifest.json +4918a44fd6c0643c984b6cba3914c53c9a3a656f313ec904e9ba7e5ed1303279 ./training-data/cpt-7b-mix/source_graph.json diff --git a/archive/legacy-hf-layout-index.md b/archive/legacy-hf-layout-index.md new file mode 100644 index 0000000000000000000000000000000000000000..52a2de865be93dcd404466e62f30c5cbf90a165d --- /dev/null +++ b/archive/legacy-hf-layout-index.md @@ -0,0 +1,27 @@ +# Legacy Hugging Face Layout Index + +The previous Hugging Face repo layout included useful but noisy top-level +folders: + +- `analysis/`; +- `artifacts/`; +- `continuous/`; +- `experiments/`; +- `fresh/`; +- `metadata/`; +- `subprojects/`; +- `tokenizers/`. + +Those paths are retained for traceability. The release entrypoints are now the +curated folders at the repo root: + +- `tokenizer/`; +- `checkpoints/`; +- `training-data/`; +- `results/`; +- `provenance/`; +- `code-links/`. + +Do not treat this file as a deletion plan. It is an index explaining the old +layout while the new additive structure is reviewed. + diff --git a/archive/legacy_SHA256SUMS_20260525 b/archive/legacy_SHA256SUMS_20260525 new file mode 100644 index 0000000000000000000000000000000000000000..041523c95c6f769c1eb05700414732104dcbb62f --- /dev/null +++ b/archive/legacy_SHA256SUMS_20260525 @@ -0,0 +1,23 @@ +e6e321ad419e32ce43427bba05a02a23133f05500a3486f98c5de232830c1844 ./README.md +5c2140f0d646a2f5eaf7ea07c033fade7e28b37468eeb8595f61f6044f6cf109 ./SHA256SUMS +f6742a090b39f1f576d18481bb1e284720275b0ab417393f219650821e83a72a ./experiments/02_1_7_intrinsic_eval_sweep_20260518/CHOSEN_CUTOFF.md +3d68bbd6e6c9e25df597f46819cbe57e336b62b8a9b5b441aadeae5346e70eb7 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/README.md +9b350211cf4f0bd964feec9504a093c1062425fb12f22534a947112f357e5c25 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/REPORT.md +60a57f77e13d8204c7728e6e5a117cb947c156c97e2c57db7de587965ada58bf ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/morphscore_greek_results.json +7dd2ae959af9504d522cf4b606c573e6cefdd71c009fe8a7269eb051382351f0 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/curated_metrics_by_slice.csv +76ad56410ada9ef2d7649d3babb757287993982e03fc98057e7289ec837c990c ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/extended_metrics_by_slice.csv +190c58ce62f8878c92499ef7da9bceb26f27a1b5f25927e6681d80df59359c79 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/metrics_by_slice.csv +f823dcc20f7cc2d7435fda6c5c09e65d704de10d37330f9ed930fdcf7f75506b ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/padded_17408_metrics.csv +9e94927189811f9c5df2e6f75c3e4ffc4738d4305a35eef2b8499c6bf15c5acf ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/our_suite_raw_gcloud/summary.json +7bbfd6c76cc88190a2334ab45af03f77e8389830e46e8dcc4ef896f7e2e3b8fc ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/extended_4metric_to_25k.png +0f472db11e1f6809b2369727de9d78e224d1c5916701141eae0c5207149d0681 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/fertility_per_in_domain_slice.png +7b90ed2c76344942c48fbb445386e16fde9e0a9af0fdbd03e96054aee1ec38d3 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/headline_4metric_in_domain.png +f72541da46b7539acca2c0fc9d641d751f0461528598e9d82651e3722681c226 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/knee_analysis.png +4f0148ff0c94a8b67e329517ab260963daa027c0ab64ac992cffe4e95c073665 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/artifacts/plots/morphscore_greek.png +763f330459d536e1ca7832dddc755e7d5f43c69605f8f8ca6c688f24c7cdf895 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/curated_padded_at_17408_manifest.json +7874469554d319addc7f113e6b5d4b7f8984e3bd4db9b3dcaa3e4e476cf392f0 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/removal_mask_at_17408.jsonl +0b3c2f295d1f14c9409b39ca2e6fc98346037a7d11a7cd931b59fcff7824b2e9 ./experiments/02_1_7_intrinsic_eval_sweep_20260518/manifests/tokeval_commit.txt +7147abc0dd32b702a399e02f47cea5576594ee6ec0d089a4e1fb6d822797aad3 ./tokenizers/c3_added_17408_curated_padded/README.md +358c249e2fb29060c6b73157d428853b0c48710deffc8ee670ab1013880946c9 ./tokenizers/c3_added_17408_curated_padded/special_tokens_map.json +358ae3f29ac17c99769d6d437339e28657d5fcaed3486f8550feed3d6adfc394 ./tokenizers/c3_added_17408_curated_padded/tokenizer.json +737b93393573376f6b4707ebfbb17688d0c358cba0daa3be12b4f9f57f9cb2ae ./tokenizers/c3_added_17408_curated_padded/tokenizer_config.json diff --git a/checkpoints/baselines/retok-3p5b-iter834/README.md b/checkpoints/baselines/retok-3p5b-iter834/README.md new file mode 100644 index 0000000000000000000000000000000000000000..03211d79911f94bde41f6026f77239c8a92fdab5 --- /dev/null +++ b/checkpoints/baselines/retok-3p5b-iter834/README.md @@ -0,0 +1,12 @@ +# ReTok Baseline, 3.5B Iter 834 + +Status: hydration pointer. + +Clariden checkpoint: + +```text +/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834 +``` + +This is the ReTok baseline for the modern-only 148480-vocab tokenizer. + diff --git a/checkpoints/baselines/retok-3p5b-iter834/hydration.md b/checkpoints/baselines/retok-3p5b-iter834/hydration.md new file mode 100644 index 0000000000000000000000000000000000000000..2a1c02010d6e8492ca480371b09edef4c7e04199 --- /dev/null +++ b/checkpoints/baselines/retok-3p5b-iter834/hydration.md @@ -0,0 +1,8 @@ +# Hydration + +On Clariden: + +```bash +ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834 +``` + diff --git a/checkpoints/baselines/retok-3p5b-iter834/manifest.json b/checkpoints/baselines/retok-3p5b-iter834/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..fe2a8d688cdd5d5d57020fc5255e59377aa643e8 --- /dev/null +++ b/checkpoints/baselines/retok-3p5b-iter834/manifest.json @@ -0,0 +1,8 @@ +{ + "artifact_id": "checkpoint.baseline-retok-3p5b-iter834", + "status": "hydration-pointer", + "tokenizer": "tokenizer.modern-greek-17408", + "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_retok/checkpoints/iter_0000834", + "iteration": 834 +} + diff --git a/checkpoints/baselines/vanilla-3p5b-iter834/README.md b/checkpoints/baselines/vanilla-3p5b-iter834/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1160a28efe7ae387f4dfa6af13f77907b28a1b82 --- /dev/null +++ b/checkpoints/baselines/vanilla-3p5b-iter834/README.md @@ -0,0 +1,13 @@ +# Vanilla Baseline, 3.5B Iter 834 + +Status: hydration pointer. + +Clariden checkpoint: + +```text +/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834 +``` + +This baseline keeps the original Apertus tokenizer and remains the heldout-BPC +reference in the 3.5B continuation. + diff --git a/checkpoints/baselines/vanilla-3p5b-iter834/hydration.md b/checkpoints/baselines/vanilla-3p5b-iter834/hydration.md new file mode 100644 index 0000000000000000000000000000000000000000..cd07fba602b335a0fefa1b0a9dc501847d9c5300 --- /dev/null +++ b/checkpoints/baselines/vanilla-3p5b-iter834/hydration.md @@ -0,0 +1,8 @@ +# Hydration + +On Clariden: + +```bash +ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834 +``` + diff --git a/checkpoints/baselines/vanilla-3p5b-iter834/manifest.json b/checkpoints/baselines/vanilla-3p5b-iter834/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..33abcd7f0616df8944f3ae42823760f775af9c98 --- /dev/null +++ b/checkpoints/baselines/vanilla-3p5b-iter834/manifest.json @@ -0,0 +1,8 @@ +{ + "artifact_id": "checkpoint.baseline-vanilla-3p5b-iter834", + "status": "hydration-pointer", + "tokenizer": "swiss-ai/Apertus-8B-2509 base tokenizer", + "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_vanilla/checkpoints/iter_0000834", + "iteration": 834 +} + diff --git a/checkpoints/td-layer11-cpt-3p5b-iter834/README.md b/checkpoints/td-layer11-cpt-3p5b-iter834/README.md new file mode 100644 index 0000000000000000000000000000000000000000..62ae95fd84cb8300de9da60ded9087c037c28072 --- /dev/null +++ b/checkpoints/td-layer11-cpt-3p5b-iter834/README.md @@ -0,0 +1,30 @@ +# TD Layer 11 CPT Checkpoint, 3.5B Iter 834 + +Status: hydration pointer. + +This is the leading extended-tokenizer checkpoint after the 3.5B continuation. + +Clariden checkpoint: + +```text +/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834 +``` + +HF-format eval copy: + +```text +/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf +``` + +Tokenizer: + +```text +../../tokenizer/modern-greek-17408/ +``` + +Result summary: + +```text +../../results/SUMMARY.md +``` + diff --git a/checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md b/checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md new file mode 100644 index 0000000000000000000000000000000000000000..a0b5619cd9aafd72441276fc27b01df5b61f964b --- /dev/null +++ b/checkpoints/td-layer11-cpt-3p5b-iter834/hydration.md @@ -0,0 +1,15 @@ +# Hydration + +This release entry is a pointer, not a checkpoint payload. + +On Clariden, use: + +```bash +ls -lh /capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834 +ls -lh /capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf +``` + +Before publishing this as loadable HF weights, verify the HF eval copy contains +the expected config, tokenizer, and safetensor shards, then upload only that +inference-ready payload. + diff --git a/checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json b/checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..0fae2fd9375cd2302c0ff571d02ebc0e826e3f92 --- /dev/null +++ b/checkpoints/td-layer11-cpt-3p5b-iter834/manifest.json @@ -0,0 +1,11 @@ +{ + "artifact_id": "checkpoint.td-layer11-cpt-3p5b-iter834", + "status": "hydration-pointer", + "tokenizer": "tokenizer.modern-greek-17408", + "source_checkpoint": "/capstor/scratch/cscs/fffoivos/runs/bakeoff/continuation_3p5b_20260524T143012Z_td_layer11/checkpoints/iter_0000834", + "hf_eval_copy": "/capstor/scratch/cscs/fffoivos/runs/eval/continuation_3p5b_20260524T143012Z_td_layer11/iter_0000834_hf", + "training_stage": "3.5B continuation", + "iteration": 834, + "source_code": "https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation" +} + diff --git a/checkpoints/td-layer11-init-r17-tp2/README.md b/checkpoints/td-layer11-init-r17-tp2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b3f21558fdebd3096f393a6e45d3bccebeb5bc48 --- /dev/null +++ b/checkpoints/td-layer11-init-r17-tp2/README.md @@ -0,0 +1,13 @@ +# TD Layer 11 Init, R17-Patched TP=2 + +Status: hydration pointer. + +Clariden checkpoint: + +```text +/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release +``` + +This is the Token Distillation layer-11 initialization in Megatron TP=2 format, +after the R17/xIELU/QK-Norm preservation patch and roundtrip verification. + diff --git a/checkpoints/td-layer11-init-r17-tp2/hydration.md b/checkpoints/td-layer11-init-r17-tp2/hydration.md new file mode 100644 index 0000000000000000000000000000000000000000..f171cfaddcb59ceb6b20576e63fe393c1bd9b888 --- /dev/null +++ b/checkpoints/td-layer11-init-r17-tp2/hydration.md @@ -0,0 +1,14 @@ +# Hydration + +On Clariden: + +```bash +ls -lh /iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release +``` + +Verification evidence: + +```text +../../provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json +``` + diff --git a/checkpoints/td-layer11-init-r17-tp2/manifest.json b/checkpoints/td-layer11-init-r17-tp2/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c4f0679d67b45e6a582b140406ad8a210fb061df --- /dev/null +++ b/checkpoints/td-layer11-init-r17-tp2/manifest.json @@ -0,0 +1,9 @@ +{ + "artifact_id": "checkpoint.td-layer11-init-r17-tp2", + "status": "hydration-pointer", + "tokenizer": "tokenizer.modern-greek-17408", + "source_checkpoint": "/iopsstor/scratch/cscs/fffoivos/token_distillation/td_full25_layer11_r17_roundtrip_2357565/megatron_tp2_r17patched/release", + "format": "Megatron torch_dist TP=2, R17 patched", + "verification": "../../provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json" +} + diff --git a/code-links/README.md b/code-links/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e517f62774e36b2deaa69d4ddf7d4db5540d6400 --- /dev/null +++ b/code-links/README.md @@ -0,0 +1,17 @@ +# Source Code Links + +The runnable implementation lives in GitHub: + +```text +https://github.com/fffoivos/glossapi-tokenizer-extension/tree/main/subprojects/03_apertus_extension_and_embedding_adaptation +``` + +This Hugging Face release keeps artifact payloads, manifests, compact evidence, +and hydration instructions. It does not mirror the full scripts tree. + +See: + +```text +github_subproject_manifest.json +``` + diff --git a/code-links/github_subproject_manifest.json b/code-links/github_subproject_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..63b7c2e42057dde47f5d04e4df85f5630d81baff --- /dev/null +++ b/code-links/github_subproject_manifest.json @@ -0,0 +1,73 @@ +{ + "repo": "https://github.com/fffoivos/glossapi-tokenizer-extension", + "branch": "main", + "source_commit": "7c67cf11eb3d31f0ddd006f7d709709037db93a1", + "subproject": "subprojects/03_apertus_extension_and_embedding_adaptation", + "script_families": [ + { + "family": "tokenizer_shipping", + "paths": [ + "03_3_cscs_experiments_kickoff/ship/apertus_greek_modern_only_148480", + "03_3_cscs_experiments_kickoff/ship/apertus_greek_extended_153600" + ], + "artifacts": [ + "tokenizer.modern-greek-17408", + "tokenizer.polytonic-plus-5120" + ] + }, + { + "family": "corpus_build", + "paths": [ + "03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md", + "03_4_implementation_experiments/init_bakeoff/corpus_build" + ], + "artifacts": [ + "dataset.cpt-7b-mix" + ] + }, + { + "family": "init_and_conversion", + "paths": [ + "03_4_implementation_experiments/init_bakeoff/arms", + "03_4_implementation_experiments/init_bakeoff/megatron_patches" + ], + "artifacts": [ + "checkpoint.td-layer11-init-r17-tp2", + "checkpoint.baseline-vanilla-3p5b-iter834", + "checkpoint.baseline-retok-3p5b-iter834" + ] + }, + { + "family": "token_distillation", + "paths": [ + "TOKEN_DISTILLATION_PLAN.md", + "03_4_implementation_experiments/init_bakeoff/token_distillation" + ], + "artifacts": [ + "checkpoint.td-layer11-init-r17-tp2" + ] + }, + { + "family": "training", + "paths": [ + "TRAINING_RECIPE.md", + "03_4_implementation_experiments/init_bakeoff/bakeoff_training" + ], + "artifacts": [ + "checkpoint.td-layer11-cpt-3p5b-iter834", + "checkpoint.baseline-vanilla-3p5b-iter834", + "checkpoint.baseline-retok-3p5b-iter834" + ] + }, + { + "family": "evaluation", + "paths": [ + "03_4_implementation_experiments/init_bakeoff/eval" + ], + "artifacts": [ + "results.continuation-3p5b" + ] + } + ] +} + diff --git a/provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json b/provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json new file mode 100644 index 0000000000000000000000000000000000000000..8c81deb34b60fdf97f84aab005c9e013424f7216 --- /dev/null +++ b/provenance/conversion-roundtrip/td_layer11_r17_roundtrip_verification.json @@ -0,0 +1,43 @@ +{ + "orig_only": [], + "trip_only": [], + "standard_max_abs_diff": 0.0, + "r17_max_abs_diff": 0.0, + "xielu_max_abs_diff": 0.0, + "qk_norm_max_abs_diff": 0.0, + "standard_changed_over_tol": [], + "r17_changed_over_tol": [], + "shape_mismatches": [], + "standard_changed_over_tol_count": 0, + "r17_changed_over_tol_count": 0, + "logits": { + "logit_max_abs_diff": 0.0, + "logit_mean_abs_diff_max": 0.0, + "per_prompt": [ + { + "prompt": "Η Αθήνα είναι η πρωτεύουσα της", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 72499, + "roundtrip_top_id": 72499, + "top_id_match": true + }, + { + "prompt": "Ο Όμηρος έγραψε", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 132093, + "roundtrip_top_id": 132093, + "top_id_match": true + }, + { + "prompt": "The capital of Greece is", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 132048, + "roundtrip_top_id": 132048, + "top_id_match": true + } + ] + } +} diff --git a/provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json b/provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json new file mode 100644 index 0000000000000000000000000000000000000000..43ffec70128e1e63b5e37e4df737a3a51d1201a5 --- /dev/null +++ b/provenance/conversion-roundtrip/vanilla_r17_roundtrip_verification.json @@ -0,0 +1,43 @@ +{ + "orig_only": [], + "trip_only": [], + "standard_max_abs_diff": 0.0, + "r17_max_abs_diff": 0.0, + "xielu_max_abs_diff": 0.0, + "qk_norm_max_abs_diff": 0.0, + "standard_changed_over_tol": [], + "r17_changed_over_tol": [], + "shape_mismatches": [], + "standard_changed_over_tol_count": 0, + "r17_changed_over_tol_count": 0, + "logits": { + "logit_max_abs_diff": 0.0, + "logit_mean_abs_diff_max": 0.0, + "per_prompt": [ + { + "prompt": "Η Αθήνα είναι η πρωτεύουσα της", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 72499, + "roundtrip_top_id": 72499, + "top_id_match": true + }, + { + "prompt": "Ο Όμηρος έγραψε", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 8329, + "roundtrip_top_id": 8329, + "top_id_match": true + }, + { + "prompt": "The capital of Greece is", + "max_abs": 0.0, + "mean_abs": 0.0, + "reference_top_id": 48214, + "roundtrip_top_id": 48214, + "top_id_match": true + } + ] + } +} diff --git a/provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md b/provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md new file mode 100644 index 0000000000000000000000000000000000000000..573e6db6055774fef609c51d1d08daa0989ab95c --- /dev/null +++ b/provenance/dataset-build/CPT_DATASET_BUILD_RUNBOOK.md @@ -0,0 +1,108 @@ +# CPT Dataset Build Runbook + +This is the repeatable path for building an Apertus CPT dataset from the +published nanochat corpus. + +## Inputs + +- Source corpus: + `fffoivos/glossapi-greek-nanochat-pretraining-dataset` +- Nanochat internal dedup bundle: + `dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata` +- Apertus hard-drop overlay: + `fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z` + at `artifacts/dedup_20260519T010924Z/cpt_final_overlay/apertus_overlap_drop_docs.parquet` + +The correct order is: + +1. Download the published nanochat source rows. +2. Hard-exclude `apertus_overlap_drop_docs.parquet`. +3. Replay nanochat internal dedup with `drop_intra_and_inter`. +4. Build the final CPT mix/export from the remaining pool. + +The order matters: if Apertus-overlapping docs are removed first, an internal +duplicate family can still keep a fresh alternate representative. + +## GCP Scratch VM + +Use a scratch VM with a large disk. A practical starting point is a +`c4-highmem-32` or larger VM with a 1-2 TB SSD/PD disk. Build artifacts should +be uploaded to Hugging Face or GCS before teardown. + +Example setup on the VM: + +```bash +sudo apt-get update +sudo apt-get install -y git python3-pip +python3 -m pip install --user duckdb pandas pyarrow huggingface_hub typer blake3 polars + +git clone https://github.com//glossapi-tokenizer-extension.git +cd glossapi-tokenizer-extension +export PYTHONPATH="$PWD" +``` + +## Download Inputs + +Do not build from the old local `/home/foivos/data/glossapi_work/hf_release_publish` +copy. Hydrate from the live HF repo so the `source_dataset` names and derived +`doc_key` values match the Apertus overlay and the `wave2` dedup bundle. + +```bash +export WORK=/mnt/disks/cpt_build +mkdir -p "$WORK" + +huggingface-cli download fffoivos/glossapi-greek-nanochat-pretraining-dataset \ + --repo-type dataset \ + --local-dir "$WORK/nanochat" \ + --include 'data/*.parquet' \ + --include 'dedup_metadata/latest.json' \ + --include 'dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata/*' + +huggingface-cli download fffoivos/apertus-c3-dedup-audit-dedup-20260519t010924z \ + --repo-type dataset \ + --local-dir "$WORK/apertus_audit" \ + --include 'artifacts/dedup_20260519T010924Z/cpt_final_overlay/*' +``` + +## Build Fresh Deduped Pool + +```bash +export DEDUP_ROOT="$WORK/nanochat/dedup_metadata/wave2_20260426_builder_metadata_v2_latest_cleaner_20260507/builder_metadata" +export APERTUS_DROP="$WORK/apertus_audit/artifacts/dedup_20260519T010924Z/cpt_final_overlay/apertus_overlap_drop_docs.parquet" +export SELECTED="$WORK/cpt/selected_after_apertus_and_internal_dedup.parquet" + +python3 -m glossapi_corpus_cli.cli mix-prepare-selected-input \ + --output-root "$WORK/nanochat" \ + --selected-input-path "$SELECTED" \ + --exclude-doc-keys-path "$APERTUS_DROP" \ + --dedup-metadata-root "$DEDUP_ROOT" \ + --dedup-action drop_intra_and_inter \ + --dedup-exact-stage strict_and_relaxed \ + --dedup-similarity-threshold 0.85 \ + --dedup-inter-dataset-policy share_aware +``` + +`$SELECTED` is the deduplicated, Apertus-fresh source pool. If the CPT recipe +uses the whole remaining pool, this is the main dataset artifact. + +## Optional Source Mix + +If the CPT run needs a sampled or weighted mix, build it from `$SELECTED`: + +```bash +python3 -m glossapi_corpus_cli.cli mix-build-from-selected-input \ + --selected-input-path "$SELECTED" \ + --mix-output-path "$WORK/cpt/final_cpt_mix.parquet" \ + --source-mix-config-path "$WORK/cpt/source_mix.json" +``` + +## Publish And Teardown + +Upload the final selected pool or mix before deleting the VM: + +```bash +huggingface-cli upload "$WORK/cpt" . --repo-type dataset +``` + +Then verify the HF repo contains the final parquet(s), summaries, and the +`cpt_final_overlay/summary.json` provenance before tearing down the VM. diff --git a/provenance/dataset-build/MIX_RECIPE.md b/provenance/dataset-build/MIX_RECIPE.md new file mode 100644 index 0000000000000000000000000000000000000000..a3e97cd8bf66c5e1c61228f1898fae46fad2467f --- /dev/null +++ b/provenance/dataset-build/MIX_RECIPE.md @@ -0,0 +1,173 @@ +# Mix recipes — bulk and anneal + +The bakeoff (and future production CPT) consumes a stream of documents +interleaved from several sources with explicit per-source weights. Per +[`../../cpt_plan.md`](../../cpt_plan.md) v0.7 §2 + §4: + +- **Bulk phase**: one shuffled-mixture stream, all corpora at target weights from token 0. This is what the bakeoff actually trains on. +- **Anneal phase**: final 10–20 % of *production* training (out of scope for the bakeoff). Mixture shifts to highest-priority subsets; replay drops from ~30 % → ~15 %; LR decays. We define the recipe here for completeness; the bakeoff never executes it. + +Both recipes are JSON, in `recipes/`. The mix builder (`mix_builder.py`) reads either and emits a JSON-lines stream. + +## Three-step build path (reviewer round-2 Blocker 3 fix) + +The CPT dataset build follows the canonical runbook at [`../../../03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md`](../../../03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md). **Order matters**: if Apertus-overlapping docs are removed first, an internal-duplicate family can still keep a fresh alternate representative. + +``` +1. bash pull_greek_corpus.sh # nanochat parquets + dedup_metadata + Apertus overlay +2. bash prepare_greek_pool.sh # → $SELECTED parquet (Apertus-drop + drop_intra_and_inter) +3. SELECTED=$WORK/cpt/selected_after_apertus_and_internal_dedup.parquet \ + python3 mix_builder.py --recipe recipes/bulk.json --target-tokens 7_000_000_000 ... +``` + +All six Greek `bulk.json` sources point at `local_parquet: ${SELECTED}` — the single post-dedup parquet — and filter by `source_dataset` value to slice into HPLT / literary / dialogue / academic / legal / dictionary buckets. The Apertus-overlap-drop is applied **upstream** in step 2 (not per-source in mix_builder), so it applies uniformly to all six categories rather than only to HPLT (the previous bug). Internal dedup with `drop_intra_and_inter` also happens in step 2 (was missing entirely before). + +## Composition (bulk recipe — what the bakeoff actually consumes) + +Working defaults per v0.7 §2 + §4 + user answers 2026-05-20: + +| Top-level bucket | Share | Sub-allocation | +|---|---:|---| +| Greek (post-Apertus-overlap-drop, post-internal-dedup) | **70 %** | see below | +| Non-Greek replay (24 languages, 3 tiers) | **24 %** | see below | +| Code | **4 %** | Intended: `bigcode/starcoderdata` / The Stack lineage (Apertus used StarCoder); 2026-05-21 CSCS fallback: `codeparrot/codeparrot-clean-train` | +| Math | **2 %** | `HuggingFaceTB/finemath` config `finemath-3plus` (Apertus stage-1 source per `submit_apertus_8b.sh:L29`) | + +> **2026-05-21 rebalance.** Replay reduced from 26 % → 24 % to free 2 % for FineMath, taken entirely from English (FW-Edu) which went from 3.9 % → 1.9 %. The other 23 replay-language weights stay at their pre-rebalance values. Greek and code shares unchanged. + +> **2026-05-21 CSCS run detail.** The live full-build recipe reads replay and FineMath from staged local single-shard parquets under `/iopsstor/scratch/cscs/fffoivos/cpt_corpus/{replay,math}`. This avoids repeated HF dataset-metadata resolution from seven concurrent shards, which hit HF's API rate limit. The Persian replay bucket uses the local FineWeb-2 `fas_Arab` fallback shard while preserving the recipe label `replay_t2_pes_Arab`. + +> **OPUS Greek-English parallel** is listed as optional in v0.7 §4.4 but **not yet in the bakeoff bulk mix** — it needs different schema handling in `mix_builder.py` (its rows are `{translation: {el: ..., en: ...}}` rather than `{text: ...}`). Deferred to a future iteration of corpus_build. + +### Greek sub-allocation (70 % of total) + +| Source | Share of Greek | HF id / filter | +|---|---:|---| +| **HPLT clean60 (fresh; Apertus-overlap overlay applied)** | 50 % of Greek = 35 % total | nanochat dataset, `source_dataset starts with HPLT__ell_Grek` + `apertus_overlap_drop_docs.parquet` exclude | +| Literary nanochat (artos-zois + Project_Gutenberg + Ekklisiastika + 1000_prwta + klasikh + Wikisource + dimodis_logotexnia + archetai) | 26 % of Greek = 18.2 % total | nanochat dataset, filter `source_dataset` ∈ literary list | +| Dialogue + textbooks (OPUS OpenSubtitles-el + openbook + Sxolika_vivlia + ert-press + istorima) | 9 % of Greek = 6.3 % total | nanochat dataset, filter | +| Academic (openarchives.gr + greek_phd + Apothetirio_Kallipos + Apothetirio_Pergamos) | 8 % of Greek = 5.6 % total | nanochat dataset, filter | +| Legal + civic (eurlex-greek-legislation + AI-team-UoA/greek_legal_code + opengov.gr-diaboyleuseis + ellinika_dedomena_europaikou_koinovouliou) | 5 % of Greek = 3.5 % total | nanochat dataset, filter | +| Dictionary + misc (modern-greek-dictionary capped + finewiki Greek half-weight + 95k_deigma + others) | 2 % of Greek = 1.4 % total | nanochat dataset, filter; modern-greek-dictionary explicitly capped per Claude-review note in `../../collegues_Apertus_plan.md` | + +The HPLT half dominates by token mass (the underlying HPLT clean60 slice is ~50 B chars at v0.7 chars/token = ~14 B tokens; the GlossAPI sub-pools combined are smaller). The recipe weights are **probabilities for the interleaver**, which is what affects the per-step mix; effective token shares track these weights up to per-source exhaustion. + +### Non-Greek replay sub-allocation (24 % of total, across 24 languages) + +Tier weights per v0.7 §4.2 (40–50 % T1 / 35–45 % T2 / 10–15 % T3); working defaults at the midpoint, scaled to the **24 % outer share** (post-2026-05-21 rebalance: English's share absorbed the 2 % shift to FineMath): + +| Tier | Languages | Share of replay | Total share | +|---|---|---:|---:| +| **T1** (8 langs, FW2-HQ where Apertus used HQ) | eng, fra, deu, ita, spa, rus, arb, cmn | ~46 % | 11.0 % | +| **T2** (11 langs, FW2 where Apertus used standard) | tur, bul, srp, ron, heb, por, pol, nld, pes, ukr, jpn | ~41 % | 9.88 % | +| **T3** (5 langs, FW2; small per-lang) | lat, hye, kat, sqi (or als), mkd | ~13 % | 3.12 % | + +Within each tier, weights are roughly equal *except* English gets ~17 % of T1 (≈ 1.9 % of total, post-rebalance) — still the largest single non-Greek language, but trimmed from 3.9 % to 1.9 % to make room for FineMath at 2 %. The remaining 7 T1 langs each stay at 1.3 % of total. + +Per-language sources: + +- T1 (Apertus quality-filtered the 20 high-resource langs at top-10 % per-language XLM-R): **`epfml/FineWeb2-HQ`** with `config_name = _