{ "_generated_by": "scripts/hf_release/generate_release_index.py", "_manifest_commit": "0a18016", "_metric_rules": { "generation_spec": "only v2 files are read; v1 code_gen and gsm8k files are ignored", "stale_eval_guard": "a metric is written only if the eval file's metadata.step equals the released checkpoint's step", "blimp_scoring": "BOS-prefixed empty-context loglikelihood, comparable within a scale" }, "counts": { "models": 291, "with_all_metrics": 106 }, "flores_common_languages": { "by_basis": { "first200/all": { "n": 214, "languages": [ "ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "als_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "arb_Latn", "arg_Latn", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "chv_Cyrl", "cjk_Latn", "ckb_Arab", "cmn_Hans", "cmn_Hant", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ekk_Latn", "ell_Grek", "eng_Latn", "epo_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "fij_Latn", "fil_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gaz_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "gug_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kaa_Latn", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khk_Cyrl", "khk_Mong", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kjh_Cyrl", "kmb_Latn", "kmr_Latn", "knc_Arab", "knc_Latn", "kor_Hang", "ktu_Latn", "lao_Laoo", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lld_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "lvs_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "mfe_Latn", "mhr_Cyrl", "min_Arab", "min_Latn", "mkd_Cyrl", "mlt_Latn", "mni_Beng", "mos_Latn", "mri_Latn", "mya_Mymr", "myv_Cyrl", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nqo_Nkoo", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pbt_Arab", "pes_Arab", "plt_Latn", "pol_Latn", "por_Latn", "prs_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Olck", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "taq_Latn", "taq_Tfng", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tha_Thai", "tir_Ethi", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tyv_Cyrl", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "vmw_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zgh_Tfng", "zsm_Latn", "zul_Latn" ], "n_models": 115 }, "full1012/all214": { "n": 214, "languages": [ "ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "als_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "arb_Latn", "arg_Latn", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "chv_Cyrl", "cjk_Latn", "ckb_Arab", "cmn_Hans", "cmn_Hant", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ekk_Latn", "ell_Grek", "eng_Latn", "epo_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "fij_Latn", "fil_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gaz_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "gug_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kaa_Latn", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khk_Cyrl", "khk_Mong", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kjh_Cyrl", "kmb_Latn", "kmr_Latn", "knc_Arab", "knc_Latn", "kor_Hang", "ktu_Latn", "lao_Laoo", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lld_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "lvs_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "mfe_Latn", "mhr_Cyrl", "min_Arab", "min_Latn", "mkd_Cyrl", "mlt_Latn", "mni_Beng", "mos_Latn", "mri_Latn", "mya_Mymr", "myv_Cyrl", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nqo_Nkoo", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pbt_Arab", "pes_Arab", "plt_Latn", "pol_Latn", "por_Latn", "prs_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Olck", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "taq_Latn", "taq_Tfng", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tha_Thai", "tir_Ethi", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tyv_Cyrl", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "vmw_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zgh_Tfng", "zsm_Latn", "zul_Latn" ], "n_models": 69 }, "full1012/trained31": { "n": 31, "languages": [ "arb_Arab", "ben_Beng", "bul_Cyrl", "cat_Latn", "ces_Latn", "cmn_Hans", "deu_Latn", "ell_Grek", "eng_Latn", "fin_Latn", "fra_Latn", "heb_Hebr", "hin_Deva", "hrv_Latn", "hun_Latn", "ind_Latn", "ita_Latn", "jpn_Jpan", "kor_Hang", "nld_Latn", "pol_Latn", "por_Latn", "ron_Latn", "rus_Cyrl", "slk_Latn", "spa_Latn", "tam_Taml", "tha_Thai", "tur_Latn", "ukr_Cyrl", "vie_Latn" ], "n_models": 105 } }, "note": "flores_mean_bpb averages over the languages in each model's own evaluation file. flores_mean_bpb_common averages over the languages present for every model on the SAME basis, named in flores_common_basis, and is the comparable column within that basis. Models on different bases are not comparable on either column: full1012 scores all 1,012 FLORES+ devtest sentences, first200 the first 200 of them." }, "models": { "full-128k-allmulti-equal-bpe": { "run": "full-128k-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63005448000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7176754255270483, "blimp_acc": 0.8189402985074629, "code_bpb_mean": 0.5210339837815584, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6553550028359902, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.056, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.05307050796057619, "flores_mean_bpb_common": 2.6553550028359902, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; allmulti_equal (30 FineWeb-2 langs equal weight, max_files=50, repeat-sampled) data; norm=none; BPE; vocab 128,256; baseline of the tail-harm result; USED - S6d.4 equal-weighting ESTABLISHED result (rho=-0.595 vs log training weight, n=31 langs)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-allmulti-equal-bpe-fresh80": { "run": "full-128k-allmulti-equal-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63005448000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7164373723666476, "flores_mean_bpb": 1.120416702997275, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.120416702997275, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-allmulti-equal-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-allmulti-equal-bpe-fresh80_blimp_code_bpb.json; no full-128k-allmulti-equal-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-allmulti-equal-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-allmulti-equal-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-allmulti-equal-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-allmulti-equal-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-allmulti-equal-bpe-fresh80-continue-fresh160": { "run": "full-128k-allmulti-equal-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 68274994560, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-allmulti-equal-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6880661910831836, "flores_mean_bpb": 1.0915218988333202, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.09152189883332, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-allmulti-equal-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-allmulti-equal-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus": { "run": "full-128k-apertus", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61775995352, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7196643583908005, "blimp_acc": 0.8191044776119404, "code_bpb_mean": 0.5261638466492652, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7683366310551216, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.086, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04094010614101592, "gsm8k_exact_match_flexible": 0.04624715693707354, "flores_mean_bpb_common": 2.7683366310551225, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "Apertus/Mistral-Nemo (off-the-shelf); n/a (off-the-shelf, Apertus-70B-2509 pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 131,072; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus-mathcode-scratch": { "run": "full-128k-apertus-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133241384072, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "apertus", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3368332422686206, "code_bpb_mean": 0.37824387350146926, "flores_mean_bpb": 3.4253459955956593, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.0, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18423047763457165, "gsm8k_exact_match_flexible": 0.22896133434420016, "flores_mean_bpb_common": 3.425345995595659, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-apertus-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#4, apertus pretok, slug `apertus`. Mistral-Nemo regex; see EXPERIMENTS_RESULTS.md \u00a73a for the RCA of the apertus-pretok MBPP code-gen breakage (real, reproducible effect, not an artifact).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus-seed42": { "run": "full-128k-apertus-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61775995352, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7211964996436109, "blimp_acc": 0.8171194029850744, "code_bpb_mean": 0.5290844455840652, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.758608145924617, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.7586081459246174, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-apertus-seed42_code_v2.json", "humaneval_pass1": "no full-128k-apertus-seed42_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-apertus-seed42_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced": { "run": "full-128k-bpe-nfc-clean-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 65667053870, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.715856631626823, "blimp_acc": 0.8210597014925377, "code_bpb_mean": 0.5226398359086579, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6516069425300484, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.092, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03790750568612585, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.651606942530047, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab 128,004; non-parity BPE baseline for the pabpe-nfc-clean-* family; also the vocab-ladder v128k anchor; USED - balanced-1B bootstrap panel (S6.12) + paper panel + S6h code analysis (n=21 main21 general-metric rows use bpe-nfc-clean-balanced's twin? no - not in main21 itself); not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-clean-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 142374413830, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.33796765404762164, "code_bpb_mean": 0.3814762402622109, "flores_mean_bpb": 3.365416060327055, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.218, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.177407126611069, "gsm8k_exact_match_flexible": 0.20394238059135708, "flores_mean_bpb_common": 3.3654160603270533, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-clean-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#5, clean pretok, slug `bpe-nfc-clean-balanced`. Clean-regex BPE (NFC, balanced data).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v128k": { "run": "full-128k-bpe-nfc-clean-balanced-v128k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70259721254, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v128k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.716152463637094, "blimp_acc": 0.8208208955223881, "code_bpb_mean": 0.5244360004512393, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644795981112231, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.086, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03639120545868082, "flores_mean_bpb_common": 2.650547974438611, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~128k target (byte-matched: 8797 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v128k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v160k": { "run": "full-128k-bpe-nfc-clean-balanced-v160k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 160000, "total_parameters": 1419118346 }, "training": { "step": 8667, "total_batch_size": 1048576, "token_budget": 9088008192, "total_bytes_consumed": 71426454399, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v160k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7142973980056193, "blimp_acc": 0.8268955223880593, "code_bpb_mean": 0.5210469295133383, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6529233096337332, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.064, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.658698154212877, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~160k target (byte-matched: 8667 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v160k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v200k": { "run": "full-128k-bpe-nfc-clean-balanced-v200k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 8552, "total_batch_size": 1048576, "token_budget": 8967421952, "total_bytes_consumed": 72714808184, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7128314584843901, "blimp_acc": 0.815268656716418, "code_bpb_mean": 0.5192765421244038, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.669924969239849, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.092, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 2.6757658908272184, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~200k target (byte-matched: 8552 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v64k": { "run": "full-128k-bpe-nfc-clean-balanced-v64k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 64000, "total_parameters": 976750346 }, "training": { "step": 9299, "total_batch_size": 1048576, "token_budget": 9750708224, "total_bytes_consumed": 66611963753, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v64k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7193331865186551, "blimp_acc": 0.8131194029850745, "code_bpb_mean": 0.5268749863541311, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6168464994006033, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.086, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.05686125852918878, "gsm8k_exact_match_flexible": 0.058377558756633814, "flores_mean_bpb_common": 2.6225363104058457, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~64k target (byte-matched: 9299 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v64k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v96k": { "run": "full-128k-bpe-nfc-clean-balanced-v96k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 96000, "total_parameters": 1124206346 }, "training": { "step": 8986, "total_batch_size": 1048576, "token_budget": 9422503936, "total_bytes_consumed": 68734120678, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v96k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7186254102526758, "blimp_acc": 0.8216865671641791, "code_bpb_mean": 0.5264406583223474, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644620729776012, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.074, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.045489006823351025, "flores_mean_bpb_common": 2.650297304590186, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~96k target (byte-matched: 8986 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v96k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-fineweb2full": { "run": "full-128k-bpe-nfc-clean-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8796, "total_batch_size": 1048576, "token_budget": 9223274496, "total_bytes_consumed": 58068658752, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127996, "metrics": { "val_bpb": 0.7249761642249047, "blimp_acc": 0.8208358208955225, "code_bpb_mean": 0.5276609173558784, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.012010693946764, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02880970432145565, "gsm8k_exact_match_flexible": 0.03184230477634572, "flores_mean_bpb_common": 3.012010693946764, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "clean-multi regex; fineweb2full (full FineWeb-2 corpus) data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-clean-fineweb2full family; USED - S5b fineweb2full PA-BPE/BPE analysis + S6e.1 cohort update; fineweb2full data variant, excluded from the main paper panel per PAPER_PANEL.md rule", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-nomark-balanced": { "run": "full-128k-bpe-nfc-clean-nomark-balanced", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70205602110, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-nomark-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7163375459110471, "blimp_acc": 0.8215223880597016, "code_bpb_mean": 0.5240401515508685, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6477544609955026, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.09, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.04473085670962851, "flores_mean_bpb_common": 2.6477544609955026, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "clean-multi regex, combining-mark variant (REGEX_CLEAN_NOMARK: \\\\p{M} removed so combining marks split from base letters; documented in EXPERIMENTS_PLAN.md XL-pretok, located 2026-08-16); balanced (D2) data; norm=NFC; BPE; vocab 128,000 (verified from tokenizer.json 2026-08-16); combining-mark control run alongside the vocab-ladder cohort (CHRONOLOGICAL 2026-07-29: 'the eleventh' run, evals complete except MGSM, later fixed); USED - companion control for the vocab-ladder cohort (11th run in that batch, CHRONOLOGICAL 2026-07-29); not itself a panels.json entry; USED - crosslingual panel v1 rev5 Tier-1 member (user-approved inclusion 2026-08-16, configs/panels/crosslingual_panel_v1.json)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-nomark-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-gpt4-fineweb2full": { "run": "full-128k-bpe-nfc-gpt4-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8796, "total_batch_size": 1048576, "token_budget": 9223274496, "total_bytes_consumed": 57262408596, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-gpt4-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127996, "metrics": { "val_bpb": 0.7240783473399763, "blimp_acc": 0.823686567164179, "code_bpb_mean": 0.5254870925448322, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9755322865422213, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.08, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 2.975532286542221, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "GPT-4o regex; fineweb2full data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-gpt4-fineweb2full family; USED - S5b fineweb2full analysis + S6e.1; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-gpt4-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced": { "run": "full-128k-bpe-nfc-plus2-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70016628000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7162315897661191, "blimp_acc": 0.8137164179104476, "code_bpb_mean": 0.5248969014124836, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.642763855645015, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.11, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.032600454890068235, "flores_mean_bpb_common": 2.6427638556450135, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "clean-multi plus2 (REGEX_CLEAN_PLUS2: attaches both apostrophe forms, contraction arm removed); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for the plus2 apostrophe-attachment pretok (ported from the Apertus-v2 fw2full family); USED - PAPER_PANEL.md paper-eligible addition (#5, landed after the S6.12 balanced-1B panel was built; paper_bootstrap_balanced_panel n=34); not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3384368951749275, "code_bpb_mean": 0.3785201421477085, "flores_mean_bpb": 3.372071226576324, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.114, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21152388172858225, "gsm8k_exact_match_flexible": 0.24184988627748294, "flores_mean_bpb_common": 3.3720712265763217, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#27, clean pretok, slug `bpe-nfc-plus2-balanced`. Standard BPE, clean_multi regex + `plus2` per-language vocab-allocation modifier, `balanced` data, vocab 128,256. Paper-eligible; landed after the balanced-1B panel (\u00a76.12 track note). FINISHED (step 19073). MBPP 0.016 [0.006, 0.028] (seed 42; 3-seed family mean 0.1047, SD 0.0782, this row is the family minimum: see \u00a73a.5).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3387550570042015, "flores_mean_bpb": 3.3593633851683737, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.212, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.23426838514025777, "flores_mean_bpb_common": 3.361894830860217, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3387273960099328, "flores_mean_bpb": 3.374212385169335, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.228, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19863532979529946, "gsm8k_exact_match_flexible": 0.24336618650492797, "flores_mean_bpb_common": 3.3766710877232637, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus3-balanced": { "run": "full-128k-bpe-nfc-plus3-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69921086400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-plus3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7149187453363852, "blimp_acc": 0.8184626865671641, "code_bpb_mean": 0.523232159118093, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.639652726658268, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.066, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.639652726658266, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "clean-multi plus3 (REGEX_CLEAN_PLUS3: plus2 + guarded trailing-apostrophe attachment); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for plus3; USED - PAPER_PANEL.md paper-eligible addition (#6); paper_bootstrap_balanced_panel n=34; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus3-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151546009194, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33869831349958357, "code_bpb_mean": 0.3834472849799035, "flores_mean_bpb": 3.3760685904096084, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.22, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19711902956785443, "gsm8k_exact_match_flexible": 0.24260803639120546, "flores_mean_bpb_common": 3.3760685904096106, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus3-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#28, clean pretok, slug `bpe-nfc-plus3-balanced`. Standard BPE, clean_multi regex + `plus3` allocation, `balanced`, vocab 128,256. Paper-eligible. FINISHED (step 19073). MBPP 0.154 [0.124, 0.186].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus3-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-allmulti-equal-bpe": { "run": "full-128k-claude-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 64445796800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7199554950159701, "blimp_acc": 0.8167611940298506, "code_bpb_mean": 0.5231038867012525, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6561368547440836, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.6561368547440845, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-claude-allmulti-equal-bpe_code_v2.json", "humaneval_pass1": "no full-128k-claude-allmulti-equal-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-claude-allmulti-equal-bpe_gsm8k_v2.json" }, "registry_note": "P3 Claude V1 regex; allmulti_equal data; norm=none; BPE; vocab 128,256; pretokenizer-generality arm, pairs with claude-balanced-bpe; USED - S6d.4 (rho=-0.558)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe": { "run": "full-128k-claude-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7139588270432892, "blimp_acc": 0.8212238805970153, "code_bpb_mean": 0.5184472344138235, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6408682996711947, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.076, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.03866565579984837, "flores_mean_bpb_common": 2.6408682996711956, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex (REGEX_CLAUDE); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-fresh80": { "run": "full-128k-claude-balanced-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7155725453770546, "flores_mean_bpb": 1.1204962764347017, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.120496276434702, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-fresh80_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-claude-balanced-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-claude-balanced-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-claude-balanced-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-fresh80-continue-fresh160": { "run": "full-128k-claude-balanced-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 75646313024, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-claude-balanced-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6847455272231185, "flores_mean_bpb": 1.0918100286280341, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.0918100286280341, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-claude-balanced-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33648727932470396, "code_bpb_mean": 0.3861471966147764, "flores_mean_bpb": 3.3743421228681676, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.25, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22744503411675512, "gsm8k_exact_match_flexible": 0.24715693707354056, "flores_mean_bpb_common": 3.3743421228681676, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#29, claude pretok, slug `claude-balanced-bpe`. Original main-panel BPE, Claude regex, `balanced`, vocab 128,256. FINISHED (step 19073). Panel-best MBPP 0.220 [0.184, 0.256] (seed 42; 3-seed family mean 0.2040, SD 0.0139: see \u00a73a.5).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed42": { "run": "full-128k-claude-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7155515391969438, "blimp_acc": 0.8163134328358209, "code_bpb_mean": 0.5202755220264306, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6456374422355053, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.082, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 2.6456374422355053, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed43": { "run": "full-128k-claude-balanced-bpe-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7159427860700754, "blimp_acc": 0.8221641791044774, "code_bpb_mean": 0.5208983240433244, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6420217727556574, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.076, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.642021772755657, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-claude-balanced-bpe-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.337272087796867, "flores_mean_bpb": 3.3792211859742856, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.234, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23199393479909022, "gsm8k_exact_match_flexible": 0.26838514025777105, "flores_mean_bpb_common": 3.3816474349880004, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed44": { "run": "full-128k-claude-balanced-bpe-seed44", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7153493010866613, "blimp_acc": 0.8171044776119403, "code_bpb_mean": 0.5204270795371622, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6489626205373287, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.648962620537327, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-claude-balanced-bpe-seed44_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed44", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33757502590578914, "flores_mean_bpb": 3.3411153775837144, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.24, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21455648218347234, "gsm8k_exact_match_flexible": 0.244882486732373, "flores_mean_bpb_common": 3.343559758978884, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe": { "run": "full-128k-claude-balanced-nfc-bpe", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807698400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7135749335882782, "blimp_acc": 0.8230895522388058, "code_bpb_mean": 0.517861350253653, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1339517058053759, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.06, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.03411675511751327, "flores_mean_bpb_common": 1.1339517058053759, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe-continue": { "run": "full-128k-claude-balanced-nfc-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 75646160448, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6997654552807139, "blimp_acc": 0.5922388059701494, "code_bpb_mean": 0.5090472908103192, "blimp_scoring": null, "flores_mean_bpb": 1.118163864176863, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.13, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.06747536012130402, "gsm8k_exact_match_flexible": 0.0712661106899166, "flores_mean_bpb_common": 1.118163864176863, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-claude-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300253589, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33714380335290617, "code_bpb_mean": 0.38451702385863634, "flores_mean_bpb": 3.3666954557164184, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.224, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2304776345716452, "gsm8k_exact_match_flexible": 0.26838514025777105, "flores_mean_bpb_common": 3.369113178784184, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#50, claude pretok, slug `claude-balanced-nfc-bpe`. Main-grid Claude NFC cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe-seed42": { "run": "full-128k-claude-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807698400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7152138612405923, "blimp_acc": 0.8188955223880598, "code_bpb_mean": 0.520628642499762, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.645825491438901, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.098, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.043214556482183475, "gsm8k_exact_match_flexible": 0.05079605761940864, "flores_mean_bpb_common": 2.645825491438902, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram": { "run": "full-128k-claude-balanced-unigram", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59884660000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7320967780165727, "blimp_acc": 0.8351641791044776, "code_bpb_mean": 0.5539673906500296, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.68308108385546, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.052, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.02880970432145565, "flores_mean_bpb_common": 2.68308108385546, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram-mathcode-scratch": { "run": "full-128k-claude-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129793195475, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3545650839336789, "code_bpb_mean": 0.4032088690906815, "flores_mean_bpb": 3.369774047069325, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.222, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1645185746777862, "gsm8k_exact_match_flexible": 0.20773313115996966, "flores_mean_bpb_common": 3.3721912339599145, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#54, claude pretok, slug `claude-balanced-unigram`. Main-grid UnigramLM cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram-seed42": { "run": "full-128k-claude-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59884660000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.735912221128107, "blimp_acc": 0.840686567164179, "code_bpb_mean": 0.5577539471842974, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6883731939103117, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.032600454890068235, "flores_mean_bpb_common": 2.688373193910312, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe": { "run": "full-128k-claude-english-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65874424000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7312714204866293, "blimp_acc": 0.8178805970149255, "code_bpb_mean": 0.5454915866651372, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6425029933386486, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.0356330553449583, "flores_mean_bpb_common": 2.642502993338647, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (same class as gpt4o-english-bpe); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe-mathcode-scratch": { "run": "full-128k-claude-english-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 142775328290, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34048861753468623, "code_bpb_mean": 0.36517357096573205, "flores_mean_bpb": 3.1992614684262377, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.234, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.221379833206975, "gsm8k_exact_match_flexible": 0.26156178923426837, "flores_mean_bpb_common": 3.2041002150765885, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-claude-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#51, claude pretok, slug `claude-english-bpe`. Main-grid Claude English-data cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe-seed42": { "run": "full-128k-claude-english-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65874424000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7329930803024799, "blimp_acc": 0.8168358208955225, "code_bpb_mean": 0.5426109795117547, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6443602201757663, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.072, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03790750568612585, "flores_mean_bpb_common": 2.6443602201757646, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-clauderev-balanced-bpe": { "run": "full-128k-clauderev-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 71005158400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "clauderev-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7095599606329199, "blimp_acc": 0.8197910447761193, "code_bpb_mean": 0.520372828134578, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6534195737157424, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.06, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.047763457164518575, "gsm8k_exact_match_flexible": 0.052312357846853674, "flores_mean_bpb_common": 2.6534195737157416, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "REGEX_CLAUDE_REV (Claude V2: no camel-case split, no cross-script split, dedicated contraction clause, lookbehind-gated space-attach); balanced (D2) data; norm=NFD + delete CR/VT/FF/NEL + fold-whitespace-to-U+0020 (Sequence normalizer); BPE; vocab 128,256; Claude V2 spec (2026-07-05 build); BPB metrics were biased +6.40% by the NFD byte-denominator bug, FIXED 2026-07-13 (accuracy metrics MBPP/HumanEval/GSM8K/BLiMP/code_bpb unaffected); never enters a val_bpb/FLORES correlation per panels.json note; USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-clauderev-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-clauderev-balanced-bpe-mathcode-scratch": { "run": "full-128k-clauderev-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153895612064, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "clauderev-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33694964163473035, "blimp_acc": 0.743537313432836, "code_bpb_mean": 0.3561613667750621, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.375535076836162, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.238, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20773313115996966, "gsm8k_exact_match_flexible": 0.2517058377558757, "flores_mean_bpb_common": 3.378083243247371, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "#26, clauderev pretok, slug `clauderev-balanced-bpe`. Claude V2: revised-Claude pretok (no camel-case/script split, lookbehind-gated space-attach, contraction-suffix clause) + NFD+CR/VT/FF/NEL-delete+whitespace-fold normalization (std-1B twin note has full pretok detail). BPE, balanced data, vocab 128,256. FINISHED step 19073: MBPP 0.206 (3rd of 37), HumanEval 0.079, GSM8K 0.240, mc_total ~0.267. INVALIDATED 2026-07-13: FLORES/val BPB understated (NFD inflates the BPB byte denominator +6.40% overall, +126.9% Korean); MBPP/HumanEval/GSM8K/mc_*/code_bpb unaffected. See EXPERIMENTAL_SETUP.md 'BPB byte-denominator bias'.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-clauderev-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt2-balanced-bpe": { "run": "full-128k-gpt2-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69381127200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt2-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.712764934305536, "blimp_acc": 0.8157761194029848, "code_bpb_mean": 0.5154497166409894, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.639768945551339, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.03, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.03411675511751327, "flores_mean_bpb_common": 2.6397689455513387, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "plain ByteLevel(use_regex=True) i.e. GPT-2 tiktoken regex; balanced (D2) data; norm=none; BPE; vocab 128,260; reference/default regex missing from the original panel; USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt2-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt2-balanced-bpe-continue": { "run": "full-128k-gpt2-balanced-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 75183912384, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "gpt2-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7000318157244623, "flores_mean_bpb": 1.1203387856616278, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1203387856616278, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt2-balanced-bpe-continue_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt2-balanced-bpe-continue_blimp_code_bpb.json; no full-128k-gpt2-balanced-bpe-continue_code_bpb.json", "mbpp_pass1": "no full-128k-gpt2-balanced-bpe-continue_code_v2.json", "humaneval_pass1": "no full-128k-gpt2-balanced-bpe-continue_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt2-balanced-bpe-continue_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt2-balanced-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt2-balanced-bpe-fresh80": { "run": "full-128k-gpt2-balanced-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69381127200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt2-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7144669781027705, "flores_mean_bpb": 1.1211954951041407, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1211954951041405, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt2-balanced-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt2-balanced-bpe-fresh80_blimp_code_bpb.json; no full-128k-gpt2-balanced-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-gpt2-balanced-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-gpt2-balanced-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt2-balanced-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt2-balanced-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160": { "run": "full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 75183912384, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "gpt2-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.683943611029401, "flores_mean_bpb": 1.0941586640167773, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.0941586640167773, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt2-balanced-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-allmulti-equal-nfc-bpe": { "run": "full-128k-gpt4o-allmulti-equal-nfc-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63006160800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-allmulti-equal-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7164581029723067, "blimp_acc": 0.8234776119402984, "code_bpb_mean": 0.5197707064555515, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.649538043661897, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.649538043661895, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; allmulti_equal data; norm=NFC; BPE; vocab 128,256; pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe; USED - S6d.4 (rho=-0.475)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-allmulti-equal-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-allmulti-equal-norepeat-bpe": { "run": "full-128k-gpt4o-allmulti-equal-norepeat-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63005448000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-allmulti-equal-norepeat-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7164844120425224, "blimp_acc": 0.8191791044776121, "code_bpb_mean": 0.519729913930031, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.651081552558217, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.651081552558217, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; allmulti_equal_norepeat (equal weights, max_files=50, NO repeat-sampling) data; norm=none; BPE; vocab 128,256; isolates repeat-sampling factor against allmulti-equal-bpe; USED - S6d.4 (rho=-0.581; repeat-sampling contributes at most ~1/4 of the tail penalty)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-allmulti-equal-norepeat-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe": { "run": "full-128k-gpt4o-balanced-bpe", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7113825837842997, "blimp_acc": 0.816850746268657, "code_bpb_mean": 0.5183171305128864, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.637541964433224, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.068, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.050037907505686124, "flores_mean_bpb_common": 2.637541964433223, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex (REGEX_GPT4O); balanced (D2) data; norm=none; BPE; vocab 128,260; project baseline tokenizer (used as the fixed tokenizer for the softcap ablation and as the mathcode-scratch #1 baseline); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-continue": { "run": "full-128k-gpt4o-balanced-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 73814256704, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6976504144166518, "blimp_acc": 0.6041044776119403, "code_bpb_mean": 0.5072590922262736, "blimp_scoring": null, "flores_mean_bpb": 1.1177910147049226, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.086, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0758150113722517, "gsm8k_exact_match_flexible": 0.08263836239575435, "flores_mean_bpb_common": 1.1177910147049224, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed42": { "run": "full-128k-gpt4o-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7131214303656023, "blimp_acc": 0.8196268656716418, "code_bpb_mean": 0.5205519366009381, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.645169501113407, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.066, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04624715693707354, "gsm8k_exact_match_flexible": 0.052312357846853674, "flores_mean_bpb_common": 2.645169501113408, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed43": { "run": "full-128k-gpt4o-balanced-bpe-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7130903979693096, "blimp_acc": 0.814850746268657, "code_bpb_mean": 0.5209780358893727, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6439124519757176, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.076, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6439124519757153, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-bpe-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3362506080752585, "flores_mean_bpb": 3.3809551460810874, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.072, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.24184988627748294, "gsm8k_exact_match_flexible": 0.27520849128127367, "flores_mean_bpb_common": 3.3832228929699526, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed44": { "run": "full-128k-gpt4o-balanced-bpe-seed44", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7139879083113999, "blimp_acc": 0.8208208955223881, "code_bpb_mean": 0.521758230655296, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644244603786841, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.068, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6442446037868432, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-bpe-seed44_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed44", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33549283645238004, "flores_mean_bpb": 3.3683389882182975, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.018, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23199393479909022, "gsm8k_exact_match_flexible": 0.2562547384382108, "flores_mean_bpb_common": 3.3704525969249524, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed45": { "run": "full-128k-gpt4o-balanced-bpe-seed45", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 45, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7139236572834594, "blimp_acc": 0.8200597014925374, "code_bpb_mean": 0.5210349997906205, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6493826719594296, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.07, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.649382671959431, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-bpe-seed45_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed45", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-maxfiles50-bpe": { "run": "full-128k-gpt4o-balanced-maxfiles50-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-maxfiles50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7115753975968808, "blimp_acc": 0.8217014925373136, "code_bpb_mean": 0.5175390204895408, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6323073931262115, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.632307393126211, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-maxfiles50-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; balanced (proportional) weights, max_files raised to 50, no repeat data; norm=none; BPE; vocab 128,256; NULL CONTROL for the max_files cap; USED - S6d.4 null control (rho=+0.004, p=0.98 - exonerates max_files as the driver)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-maxfiles50-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70": { "run": "full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-maxfiles50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7088121845173194, "flores_mean_bpb": 1.1201184098240862, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1201184098240864, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_blimp_code_bpb.json; no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160": { "run": "full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 73814256704, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160" }, "tokenizer_slug": "gpt4o-balanced-maxfiles50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6817898659356111, "flores_mean_bpb": 1.0914107724657318, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.0914107724657316, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-maxfiles50-bpe-fresh70-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch": { "run": "full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch", "works": [ "paper_a" ], "training_regime": "token-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-maxfiles50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7144892210222866, "flores_mean_bpb": 1.1351939844134402, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.13519398441344, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_blimp_code_bpb.json; no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-maxfiles50-bpe-tokenmatch", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe": { "run": "full-128k-gpt4o-balanced-nfc-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7114993123173433, "blimp_acc": 0.8128955223880601, "code_bpb_mean": 0.5185776938972276, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.635370935171478, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04852160727824109, "gsm8k_exact_match_flexible": 0.053828658074298714, "flores_mean_bpb_common": 2.635370935171479, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-fresh80": { "run": "full-128k-gpt4o-balanced-nfc-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7132701914103621, "flores_mean_bpb": 1.118988205255762, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.118988205255762, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80_blimp_code_bpb.json; no full-128k-gpt4o-balanced-nfc-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160": { "run": "full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 73815668032, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6826851413074535, "flores_mean_bpb": 1.0919341679005787, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.0919341679005792, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147639076801, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3360314499008519, "code_bpb_mean": 0.3786237132757522, "flores_mean_bpb": 3.4023914971799996, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.12, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20773313115996966, "gsm8k_exact_match_flexible": 0.2441243366186505, "flores_mean_bpb_common": 3.4023914971799996, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#2, gpt4o pretok, slug `gpt4o-balanced-nfc-bpe`. NFC pair to (1) for the digit-direction test.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-seed42": { "run": "full-128k-gpt4o-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7133388823384232, "blimp_acc": 0.8202089552238806, "code_bpb_mean": 0.520749213831268, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6415260365059634, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.05, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04700530705079606, "gsm8k_exact_match_flexible": 0.05307050796057619, "flores_mean_bpb_common": 2.641526036505964, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-seed43": { "run": "full-128k-gpt4o-balanced-nfc-bpe-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7131382980330084, "blimp_acc": 0.816865671641791, "code_bpb_mean": 0.5205335169217197, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.643185708082578, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.056, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6431857080825765, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-nfc-bpe-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-seed44": { "run": "full-128k-gpt4o-balanced-nfc-bpe-seed44", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137879652747063, "blimp_acc": 0.8208656716417913, "code_bpb_mean": 0.5214139991062939, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6454051197076214, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.078, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6454051197076227, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-nfc-bpe-seed44_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-seed44", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram": { "run": "full-128k-gpt4o-balanced-unigram", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7314636077017965, "blimp_acc": 0.8325522388059705, "code_bpb_mean": 0.5537964877519875, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6837907462932735, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.066, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.0310841546626232, "flores_mean_bpb_common": 2.683790746293272, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF UnigramTrainer, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-bytematched": { "run": "full-128k-gpt4o-balanced-unigram-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 12942, "total_batch_size": 1048576, "token_budget": 13570670592, "total_bytes_consumed": 88136987184, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7178646641454898, "flores_mean_bpb": 1.1578610415108015, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1578610415108013, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-unigram-bytematched_blimp_code_bpb.json; no full-128k-gpt4o-balanced-unigram-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-unigram-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-unigram-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-continue": { "run": "full-128k-gpt4o-balanced-unigram-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 64941609472, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7147524048612938, "flores_mean_bpb": 1.1551748427433306, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1551748427433306, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-continue_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-unigram-continue_blimp_code_bpb.json; no full-128k-gpt4o-balanced-unigram-continue_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-unigram-continue_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-unigram-continue_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-continue_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-fresh80": { "run": "full-128k-gpt4o-balanced-unigram-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7317820538682923, "flores_mean_bpb": 1.1571602607234888, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1571602607234885, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-unigram-fresh80_blimp_code_bpb.json; no full-128k-gpt4o-balanced-unigram-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-unigram-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-unigram-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160": { "run": "full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 64941609472, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.704182188110991, "flores_mean_bpb": 1.1248227870927645, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1248227870927647, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129890029096, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3533980943471727, "code_bpb_mean": 0.3993466898373258, "flores_mean_bpb": 3.4055634092286033, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.0, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1865049279757392, "gsm8k_exact_match_flexible": 0.23199393479909022, "flores_mean_bpb_common": 3.40792977618308, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#53, gpt4o pretok, slug `gpt4o-balanced-unigram`. Main-grid UnigramLM cell (first UnigramLM run at 20B). FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-seed42": { "run": "full-128k-gpt4o-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7337511969511488, "blimp_acc": 0.8338208955223879, "code_bpb_mean": 0.5569454278048204, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6918705524709803, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.04, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03866565579984837, "flores_mean_bpb_common": 2.6918705524709816, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-seed43": { "run": "full-128k-gpt4o-balanced-unigram-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.73386423181875, "blimp_acc": 0.8246119402985075, "code_bpb_mean": 0.5569616315916492, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6892189384539167, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.054, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6892189384539162, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-seed44": { "run": "full-128k-gpt4o-balanced-unigram-seed44", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7338959687025359, "blimp_acc": 0.8302686567164179, "code_bpb_mean": 0.5571008957487255, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.689204251359402, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.034, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6892042513594023, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-unigram-seed44_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-seed44", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-tuned": { "run": "full-128k-gpt4o-balanced-unigram-tuned", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68301156000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram-tuned", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7332765694658377, "blimp_acc": 0.838014925373134, "code_bpb_mean": 0.5564997843345418, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6912106703315612, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.042, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.028051554207733132, "gsm8k_exact_match_flexible": 0.032600454890068235, "flores_mean_bpb_common": 2.6912106703315617, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF, UNIGRAM_TUNED_KWARGS: shrinking_factor=0.7, n_sub_iterations=3, max_piece_length=64, full byte initial_alphabet); vocab 128,260; tests whether 'Unigram loses' survives tuned HF UnigramTrainer hyperparameters; USED - balanced-1B bootstrap panel (S6.12, n=32/34) and paper_bootstrap_balanced_panel; not in panels.json main21 (superseded by default-kwargs gpt4o-balanced-unigram in the settled panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-tuned", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe": { "run": "full-128k-gpt4o-code-bpe", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59664431200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7238738916126876, "blimp_acc": 0.821, "code_bpb_mean": 0.5431368524901159, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1568764640121976, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.054, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04473085670962851, "flores_mean_bpb_common": 1.1568764640121976, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; code-heavy (D3) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe-continue": { "run": "full-128k-gpt4o-code-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 64654547264, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7093935263077553, "blimp_acc": 0.5940895522388059, "code_bpb_mean": 0.5454133776172693, "blimp_scoring": null, "flores_mean_bpb": 1.1309524396542419, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.062, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0621683093252464, "gsm8k_exact_match_flexible": 0.06292645943896892, "flores_mean_bpb_common": 1.1309524396542416, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-code-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129315874577, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33264187797540906, "code_bpb_mean": 0.3804701231471243, "flores_mean_bpb": 3.152441006253198, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.016, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.25549658832448824, "flores_mean_bpb_common": 3.152441006253197, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-code-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#30, gpt4o pretok, slug `gpt4o-code-bpe`. GPT-4o regex, code-heavy tokenizer-training data, vocab 128,256. FINISHED (step 19073). MBPP 0.016 [0.006, 0.028].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe-seed42": { "run": "full-128k-gpt4o-code-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59664431200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7252439607412663, "blimp_acc": 0.8190149253731343, "code_bpb_mean": 0.5446616749526122, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6299392975518514, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.08, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.0356330553449583, "flores_mean_bpb_common": 2.629939297551851, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153517795007, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad00-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.339566757938572, "code_bpb_mean": 0.35975839197882636, "flores_mean_bpb": 3.357431043588078, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.232, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20545868081880211, "gsm8k_exact_match_flexible": 0.23805913570887036, "flores_mean_bpb_common": 3.3597134138517863, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#39, gpt4o pretok, slug `gpt4o-codegrad00-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 0%. Fused punct+newline vocab tokens: 482. FINISHED. MBPP 0.166 (generation_spec v1); v2 re-derivation (2026-08-04) 0.232, see \u00a73a.3. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad00-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151162393945, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad05-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3365866322863168, "code_bpb_mean": 0.380531779639667, "flores_mean_bpb": 3.3960370708190832, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.05, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.24564063684609552, "flores_mean_bpb_common": 3.398316435686632, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#40, gpt4o pretok, slug `gpt4o-codegrad05-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 5%. Fused 749. FINISHED. MBPP 0.048 (v1); v2 0.050. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad05-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143650893428, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad30-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33496232135285, "code_bpb_mean": 0.37663071795054853, "flores_mean_bpb": 3.3437376646164667, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.178, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20545868081880211, "gsm8k_exact_match_flexible": 0.25094768764215314, "flores_mean_bpb_common": 3.345849153542006, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#41, gpt4o pretok, slug `gpt4o-codegrad30-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 30%. Fused 1393. FINISHED. MBPP 0.048 (v1); v2 0.178. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad30-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 138855769571, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33476615801535653, "code_bpb_mean": 0.3764101019719301, "flores_mean_bpb": 3.3840800213850546, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.088, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.265352539802881, "flores_mean_bpb_common": 3.386337762228807, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#42, gpt4o pretok, slug `gpt4o-codegrad50-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 50%. Fused 1831. FINISHED. MBPP 0.108 (v1); v2 0.088. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad50-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe": { "run": "full-128k-gpt4o-english-bpe", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65128632800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.735036299787199, "blimp_acc": 0.8158358208955222, "code_bpb_mean": 0.5583797506682657, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1705711724596686, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.074, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02577710386656558, "gsm8k_exact_match_flexible": 0.030326004548900682, "flores_mean_bpb_common": 1.1705711724596686, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (drops 32 control bytes incl. '{' and DEL); see gpt4o-english-fullbyte-bpe control; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe-continue": { "run": "full-128k-gpt4o-english-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 70575754816, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7252800230078966, "flores_mean_bpb": 1.14611029813195, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1461102981319498, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-bpe-continue_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-english-bpe-continue_blimp_code_bpb.json; no full-128k-gpt4o-english-bpe-continue_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-english-bpe-continue_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-english-bpe-continue_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-english-bpe-continue_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-english-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141158910613, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.342942884295594, "code_bpb_mean": 0.37086217884139866, "flores_mean_bpb": 3.1757044421045038, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.212, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20621683093252463, "gsm8k_exact_match_flexible": 0.23275208491281274, "flores_mean_bpb_common": 3.1757044421045046, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#31, gpt4o pretok, slug `gpt4o-english-bpe`. GPT-4o regex, English-only tokenizer-training data, vocab 128,256. FINISHED (step 19073). MBPP 0.210 [0.174, 0.246].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe-seed42": { "run": "full-128k-gpt4o-english-bpe-seed42", "works": [ "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65128632800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7374689939971976, "blimp_acc": 0.8065074626865676, "code_bpb_mean": 0.5566762465787464, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6384577900595425, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.06, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.030326004548900682, "flores_mean_bpb_common": 2.644244087469852, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe": { "run": "full-128k-gpt4o-english-fullbyte-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65111437600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7382166396904717, "blimp_acc": 0.8167910447761192, "code_bpb_mean": 0.5567630102431164, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6482273537289984, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.056, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03335860500379075, "gsm8k_exact_match_flexible": 0.037149355572403335, "flores_mean_bpb_common": 2.6482273537289993, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE (trainer_kwargs full_byte_alphabet=True: initial_alphabet forced to the complete 256-entry ByteLevel alphabet); vocab 128,256; isolates the initial_alphabet byte-drop defect vs gpt4o-english-bpe (209/256 -> 256/256 bytes present; 47 fewer merges); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe-fresh80": { "run": "full-128k-gpt4o-english-fullbyte-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65111437600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7336344025476245, "flores_mean_bpb": 1.1589415505410425, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1589415505410423, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80_blimp_code_bpb.json; no full-128k-gpt4o-english-fullbyte-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160": { "run": "full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 70557121472, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7096793588767049, "flores_mean_bpb": 1.1245784748394025, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1245784748394023, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141121641971, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34344991506677197, "code_bpb_mean": 0.3990911179451489, "flores_mean_bpb": 3.182336686454046, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.064, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2175890826383624, "gsm8k_exact_match_flexible": 0.23351023502653526, "flores_mean_bpb_common": 3.1871322797732247, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe": { "run": "full-128k-gpt4o-invert-full-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9980, "total_batch_size": 1048576, "token_budget": 10464788480, "total_bytes_consumed": 104785119780, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7197210053821276, "blimp_acc": 0.8192985074626866, "code_bpb_mean": 0.5196511508328988, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1630618904452958, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.163061890445296, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-fresh70": { "run": "full-128k-gpt4o-invert-full-bpe-fresh70", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9980, "total_batch_size": 1048576, "token_budget": 10464788480, "total_bytes_consumed": 104785119780, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7138122922125957, "flores_mean_bpb": 1.132263294706739, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.132263294706739, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-fresh70_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-fresh70_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-fresh70_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-fresh70_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-fresh70_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-fresh70_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-fresh70", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160": { "run": "full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 100123336896, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6961500539034952, "flores_mean_bpb": 1.1065660857328525, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1065660857328525, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-seed43": { "run": "full-128k-gpt4o-invert-full-bpe-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9980, "total_batch_size": 1048576, "token_budget": 10464788480, "total_bytes_consumed": 104785119780, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7196134547647451, "blimp_acc": 0.8184925373134332, "code_bpb_mean": 0.5223069315606151, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1639346664178052, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1639346664178052, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-seed43-fresh70": { "run": "full-128k-gpt4o-invert-full-bpe-seed43-fresh70", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9980, "total_batch_size": 1048576, "token_budget": 10464788480, "total_bytes_consumed": 104785119780, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7133737735311857, "flores_mean_bpb": 1.132476234439329, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1324762344393293, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-seed43-fresh70", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160": { "run": "full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 100123336896, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6971419221333996, "flores_mean_bpb": 1.1086941785469064, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1086941785469067, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-tokenmatch": { "run": "full-128k-gpt4o-invert-full-bpe-tokenmatch", "works": [ "paper_a" ], "training_regime": "token-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 92395696800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7241881914260614, "flores_mean_bpb": 1.1601604914321189, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1601604914321189, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-tokenmatch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-tokenmatch_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-tokenmatch_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-tokenmatch_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-tokenmatch_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-tokenmatch_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-tokenmatch", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh": { "run": "full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh", "works": [ "paper_a" ], "training_regime": "token-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 92395696800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-invert-full-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7188714202580979, "flores_mean_bpb": 1.1223999182792723, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1223999182792723, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_blimp_code_bpb.json; no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_code_bpb.json", "mbpp_pass1": "no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-invert-full-bpe-tokenmatch-fresh", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-no-english-bpe": { "run": "full-128k-gpt4o-no-english-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9087, "total_batch_size": 1048576, "token_budget": 9528410112, "total_bytes_consumed": 75918740898, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-no-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137480879376475, "blimp_acc": 0.8183880597014926, "code_bpb_mean": 0.5215455778809348, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.143894961183905, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.143894961183905, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-no-english-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-no-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-no-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-no-english-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-perm-a-bpe": { "run": "full-128k-gpt4o-perm-a-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9692, "total_batch_size": 1048576, "token_budget": 10162798592, "total_bytes_consumed": 81976787172, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-perm-a-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7200436848567202, "blimp_acc": 0.818820895522388, "code_bpb_mean": 0.5222599341780472, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.147041380522285, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1470413805222852, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-perm-a-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-perm-a-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-perm-a-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-perm-a-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-perm-b-bpe": { "run": "full-128k-gpt4o-perm-b-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9693, "total_batch_size": 1048576, "token_budget": 10163847168, "total_bytes_consumed": 68570462925, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-perm-b-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7214014231437861, "blimp_acc": 0.8212985074626863, "code_bpb_mean": 0.5235094270600997, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1535697464822194, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1535697464822192, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-perm-b-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-perm-b-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-perm-b-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-perm-b-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.337055151819168, "code_bpb_mean": 0.38298562106914746, "flores_mean_bpb": 3.3435001896408045, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.224, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2259287338893101, "gsm8k_exact_match_flexible": 0.2585291887793783, "flores_mean_bpb_common": 3.3458386224790657, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#47, gpt4onl pretok, slug `gpt4onl-balanced-bpe`. AXES-2 single-mechanism pair: non-fusing regex on the balanced composition, fused 0. FINISHED. v1: MBPP 0.158, HumanEval 0.0183, GSM8K 0.242. Fusing counterpart is the unslugged baseline `full-128k-mathcode-scratch` (gpt4o-balanced-bpe, same recipe/step). Pair v1: MBPP 0.024 vs 0.158, HumanEval 0.0854 vs 0.0183 - benchmarks disagree on sign; both have clean prompt boundaries (\u00a73a.6), not the HumanEval artifact. v2 (2026-08-04): this run MBPP 0.224/HumanEval 0.1890; baseline MBPP 0.174/HumanEval 0.1646. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3376554366247921, "flores_mean_bpb": 3.349377121305643, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.246, "humaneval_pass1": 0.1951219512195122, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22971948445792267, "gsm8k_exact_match_flexible": 0.25549658832448824, "flores_mean_bpb_common": 3.3514450001438205, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33694131565842034, "flores_mean_bpb": 3.3315847422203544, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.22, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.2441243366186505, "flores_mean_bpb_common": 3.3338667103615087, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153764637773, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad00-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34202783106124746, "code_bpb_mean": 0.36323765518867457, "flores_mean_bpb": 3.351781915121498, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.248, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21076573161485973, "gsm8k_exact_match_flexible": 0.24336618650492797, "flores_mean_bpb_common": 3.3543972522639045, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#43, gpt4onl pretok, slug `gpt4onl-codegrad00-bpe`. AXES-2 code-fraction gradient, non-fusing regex (REGEX_GPT4O_NONL: punct arm loses trailing [\\r\\n/]*, else byte-identical to REGEX_GPT4O), tokenizer-corpus code fraction 0%. Fused 0 by construction. FINISHED. MBPP 0.232 (generation_spec v1), the highest of any run in the 20B mathcode family (v1-scored, like-for-like with the panel). v2 re-derivation (2026-08-04) 0.248, see \u00a73a.3. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151521519462, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad05-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3385269818145197, "code_bpb_mean": 0.38338144733764795, "flores_mean_bpb": 3.3565886166875782, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.206, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21228203184230476, "gsm8k_exact_match_flexible": 0.24639878695981804, "flores_mean_bpb_common": 3.358597316235766, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#44, gpt4onl pretok, slug `gpt4onl-codegrad05-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 5%. Fused 0. FINISHED. MBPP 0.204 (v1); v2 0.206. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 144184498749, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad30-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3366932052292442, "code_bpb_mean": 0.3807316548423415, "flores_mean_bpb": 3.3328335389250148, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.224, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2001516300227445, "gsm8k_exact_match_flexible": 0.2312357846853677, "flores_mean_bpb_common": 3.3348439114890582, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#45, gpt4onl pretok, slug `gpt4onl-codegrad30-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 30%. Fused 0. FINISHED. MBPP 0.178 (v1); v2 0.224. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 139282241851, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3365895514079721, "code_bpb_mean": 0.3791457119657013, "flores_mean_bpb": 3.3430497401898602, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.212, "humaneval_pass1": 0.1951219512195122, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2350265352539803, "gsm8k_exact_match_flexible": 0.26004548900682334, "flores_mean_bpb_common": 3.3454704584985673, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#46, gpt4onl pretok, slug `gpt4onl-codegrad50-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 50%. Fused 0. FINISHED. MBPP 0.174 (v1); v2 0.212. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highmid-bpe": { "run": "full-128k-highmid-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69171696000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highmid-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137029022782296, "blimp_acc": 0.8148805970149254, "code_bpb_mean": 0.5213740689827874, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6410043621507118, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.096, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.058377558756633814, "flores_mean_bpb_common": 2.6410043621507113, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highmid (21 high+mid-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 21 langs); USED - S6d.3 coverage ladder; superseded within panels.json main21 by highmid-unigram", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-highmid-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highmid-unigram": { "run": "full-128k-highmid-unigram", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 61515159200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highmid-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7345206818002139, "blimp_acc": 0.8309552238805971, "code_bpb_mean": 0.5576028102921461, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.679185239111264, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02880970432145565, "gsm8k_exact_match_flexible": 0.032600454890068235, "flores_mean_bpb_common": 2.679185239111264, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highmid (21 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highmid-bpe; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-highmid-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highres-bpe": { "run": "full-128k-highres-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 76247124800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highres-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7127224232645878, "blimp_acc": 0.8137164179104476, "code_bpb_mean": 0.5216821970561181, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6217592725614316, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.046, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.034874905231235785, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 2.6217592725614325, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highres (6 high-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 6 langs, LM trains on all 30); USED - S6d.3 coverage ladder (english-only/highres/highmid/balanced GPT-4o-BPE comparison); superseded within panels.json main21 by highres-unigram", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-highres-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highres-unigram": { "run": "full-128k-highres-unigram", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 67182544000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highres-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7305499098474624, "blimp_acc": 0.8239552238805969, "code_bpb_mean": 0.5556786069357298, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.669595217297764, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.072, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.026535253980288095, "gsm8k_exact_match_flexible": 0.030326004548900682, "flores_mean_bpb_common": 2.669595217297765, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highres (6 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highres-bpe, completes the {BPE,Unigram}x{highres,highmid,balanced} grid; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-highres-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3": { "run": "full-128k-llama3", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59564639200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.718139704096831, "blimp_acc": 0.8201940298507464, "code_bpb_mean": 0.5477978759768082, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1472750609962472, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.052, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 1.1472750609962472, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": "LLaMA-3 tiktoken (off-the-shelf); n/a (off-the-shelf, Meta-Llama-3-8B pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 128,256; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3-continue": { "run": "full-128k-llama3-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 64546409024, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7038178794091192, "blimp_acc": 0.5893283582089552, "code_bpb_mean": 0.5596346024585033, "blimp_scoring": null, "flores_mean_bpb": 1.1253121937438444, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.09, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.060652009097801364, "gsm8k_exact_match_flexible": 0.0667172100075815, "flores_mean_bpb_common": 1.1253121937438444, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-llama3-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3-mathcode-scratch": { "run": "full-128k-llama3-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129099586757, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "llama3", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.3334353889147577, "code_bpb_mean": 0.3778736514478963, "flores_mean_bpb": 3.3787162974554765, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.082, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23275208491281274, "gsm8k_exact_match_flexible": 0.2721758908263836, "flores_mean_bpb_common": 3.378716297455475, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-llama3-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#32, llama3 pretok, slug `llama3`. Off-the-shelf Meta-Llama-3 tokenizer, vocab 128,256. FINISHED (step 19073). MBPP 0.166 [0.134, 0.198].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-llama3-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3-seed42": { "run": "full-128k-llama3-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59564639200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7197870317421321, "blimp_acc": 0.8257014925373133, "code_bpb_mean": 0.5493603027008278, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.66581772189602, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.082, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.05155420773313116, "flores_mean_bpb_common": 2.6658177218960195, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-llama3-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-mathcode-scratch": { "run": "full-128k-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33515701265710024, "blimp_acc": 0.5596268656716418, "code_bpb_mean": 0.37733489353750416, "blimp_scoring": null, "flores_mean_bpb": 3.2665762921686197, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.174, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19787717968157695, "gsm8k_exact_match_flexible": 0.22896133434420016, "flores_mean_bpb_common": 3.266576292168622, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "#1, gpt4o pretok, slug `gpt4o-balanced-bpe`. Original baseline (unprefixed).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61590124482, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.728561092002857, "blimp_acc": 0.8191194029850744, "code_bpb_mean": 0.531266649961596, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9432145980087117, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.088, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03790750568612585, "gsm8k_exact_match_flexible": 0.039423805913570885, "flores_mean_bpb_common": 2.9432145980087125, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, apertus/Mistral-Nemo pretok regex; fineweb2full, capped + tuned data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; 2x2 candidate; the apertus-pretok arm later diagnosed with the MBPP-collapse RCA (fuses code line-endings into superwords) at 20B scale; USED - Apertus-2 development track candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133580673669, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.3444556663811808, "code_bpb_mean": 0.38378437135843607, "flores_mean_bpb": 3.485736844563002, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.182, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21152388172858225, "gsm8k_exact_match_flexible": 0.23730098559514784, "flores_mean_bpb_common": 3.4857368445630046, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#8, apertus pretok, slug `pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow`. PA-BPE candidate on apertus pretok.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-balanced": { "run": "full-128k-pabpe-nfc-clean-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127836, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72926909582, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127832, "metrics": { "val_bpb": 0.7234690685527866, "blimp_acc": 0.8185820895522389, "code_bpb_mean": 0.527456023839792, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6582937031147855, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.082, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.03639120545868082, "flores_mean_bpb_common": 2.6582937031147846, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with clean-multi regex; balanced (D2) data; norm=NFC; PA-BPE (base); vocab 127,836; same data as pabpe-nfc-gpt4-balanced (2a), different pretok regex (controlled pretok ablation); bug-fixed 2026-05-16 replacement (2b) for retired pabpe-balanced; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-balanced-continue": { "run": "full-128k-pabpe-nfc-clean-balanced-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127836, "total_parameters": 1271072522 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 79080169408, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "pabpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127832, "metrics": { "val_bpb": 0.7090211796375733, "flores_mean_bpb": 1.1471034298012621, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.147103429801262, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-balanced-continue_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-balanced-continue_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-balanced-continue_code_bpb.json", "mbpp_pass1": "no full-128k-pabpe-nfc-clean-balanced-continue_code_v2.json", "humaneval_pass1": "no full-128k-pabpe-nfc-clean-balanced-continue_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-pabpe-nfc-clean-balanced-continue_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-balanced-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127836, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152278889219, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-balanced-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127832, "metrics": { "val_bpb": 0.3407285888388783, "code_bpb_mean": 0.3791105534182907, "flores_mean_bpb": 3.3507237031110604, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.232, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21304018195602728, "gsm8k_exact_match_flexible": 0.2312357846853677, "flores_mean_bpb_common": 3.35072370311106, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#6, clean pretok, slug `pabpe-nfc-clean-balanced-hybridwindow`. PA-BPE on clean pretok (balanced, HW).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full": { "run": "full-128k-pabpe-nfc-clean-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72208518928, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7485355018717734, "blimp_acc": 0.8243134328358208, "code_bpb_mean": 0.5609107321473775, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.8514843111277366, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.056, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.022744503411675512, "gsm8k_exact_match_flexible": 0.025018953752843062, "flores_mean_bpb_common": 2.8514843111277357, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,835; USED - S5b fineweb2full analysis (best all-FLORES BPB 2.848 but worst val_bpb/BLiMP/code_bpb of the six); fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61636354540, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7288989708540804, "blimp_acc": 0.815955223880597, "code_bpb_mean": 0.5329979934179307, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9645154627226296, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.078, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03790750568612585, "flores_mean_bpb_common": 2.964515462722632, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi regex; fineweb2full, capped ({1,16} repeat-run cap) + quota-tuned ('tuned'/v5 hand-tuned per-family weighting) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; one of 4 Apertus-2-track candidates in the original 2x2 (pretok x {apertus,clean}) design; USED - Apertus-2 development track candidate; excluded from main paper panel (fineweb2full data variant, PAPER_PANEL.md rule)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133680940430, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34503526842456766, "code_bpb_mean": 0.38495957385019997, "flores_mean_bpb": 3.4963244629478334, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.234, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17589082638362397, "gsm8k_exact_match_flexible": 0.2047005307050796, "flores_mean_bpb_common": 3.4963244629478334, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#7, clean pretok, slug `pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow`. PA-BPE candidate (fw2full, capped-tuned).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60615581520, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7253181858942936, "blimp_acc": 0.8196119402985071, "code_bpb_mean": 0.5292204446848089, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.982115075242416, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.078, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.037149355572403335, "flores_mean_bpb_common": 2.9821150752424175, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 engfull_eu3 (full English FineWeb-1 sample + European boost + Arabic fix) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_enh' (English-preserving); roster #22; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130738548720, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34234477191826124, "code_bpb_mean": 0.38480415557318853, "flores_mean_bpb": 3.5552682701866014, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.224, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21228203184230476, "gsm8k_exact_match_flexible": 0.2350265352539803, "flores_mean_bpb_common": 3.555268270186603, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#22, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k`. **Apertus-v2 candidate** `preliminary_enh` (English-preserving: English-boosted + moderate EU + Arabic fix). consv2 `engfull_eu3` grouped config, gm110k, vocab 131072 + 124 specials, TemplateProcessing post-proc (suppressed in nanochat path). Source `\u2026_consv2_engfull_eu3_gm110k_v131k_sp124_eng5g`. Std-1B 2571385+, mathcode 2571425+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60085213752, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7273522461094571, "blimp_acc": 0.8184179104477616, "code_bpb_mean": 0.5323586574775419, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.958214536949256, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.062, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.03335860500379075, "flores_mean_bpb_common": 2.9582145369492565, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 + eudata data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; same eudata config, higher global-merges warmup; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61317149768, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7276288753194399, "blimp_acc": 0.8188059701492535, "code_bpb_mean": 0.5322564133385435, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9892469043285885, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.088, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.037149355572403335, "flores_mean_bpb_common": 2.989246904328589, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi regex, plus2/repcap8/capped/tuned base; fineweb2full, consv2 quota formula + eudata (European-data-quota grouped config) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (131,072 target + 124 specials); earlier eudata sweep point, superseded by the shipped consv2 candidates; USED - S6f Contrast B (PA-BPE consv2 family, 131k arm, n=12) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60642676472, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7261196904187828, "blimp_acc": 0.8146865671641791, "code_bpb_mean": 0.5310156660973219, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9639366438466745, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.078, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.034874905231235785, "flores_mean_bpb_common": 2.963936643846674, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm100k), clean-multi regex; fineweb2full, consv2 + eudata4 variant data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 9903, "total_batch_size": 1048576, "token_budget": 10384048128, "total_bytes_consumed": 74440593522, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7201628703050716, "blimp_acc": 0.8205522388059703, "code_bpb_mean": 0.523890471891906, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.069992442307267, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.068, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 3.069992442307267, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm130k), clean-multi regex; fineweb2full, consv2 eusino_v2c + French/German boost + kr120 (+120MB Korean) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,064; the shipped preliminary_mul_200k recipe; roster #38; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143371245102, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3414332492357774, "code_bpb_mean": 0.38486892005448553, "flores_mean_bpb": 3.584565348888925, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.228, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2100075815011372, "gsm8k_exact_match_flexible": 0.2441243366186505, "flores_mean_bpb_common": 3.5845653488889253, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#38, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k`. PA-BPE Apertus-v2 candidate, consv2 `eusino_v2c` + frde + kr120, gm130k, vocab 200,000 (md5 3cdc3bdd). Distinct tokenizer from roster #25 (`\u2026-eusino-v2c-gm120k-v200k`, gm120k, md5 a5bdcfae). FINISHED (step 19073). MBPP 0.206 [0.172, 0.242].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 9903, "total_batch_size": 1048576, "token_budget": 10384048128, "total_bytes_consumed": 74997795720, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7201429837377527, "blimp_acc": 0.8141194029850746, "code_bpb_mean": 0.5238153919083227, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.1039007372839995, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.076, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0401819560272934, "gsm8k_exact_match_flexible": 0.043214556482183475, "flores_mean_bpb_common": 3.1113714246218653, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm120k), clean-multi regex; fineweb2full, consv2 eusino_v2c (European+Sinotibetan rebalance) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,000; the 200k predecessor LM evaluated for shipped candidate 'preliminary_mul_200k'; roster #25; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 144444406520, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34148051897222004, "code_bpb_mean": 0.3853258864786902, "flores_mean_bpb": 3.5718106322903322, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.18, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.22289613343442002, "flores_mean_bpb_common": 3.5718106322903345, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#25, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k`. **Apertus-v2 candidate** `preliminary_mul_200k` (200k all-rounder). consv2 `eusino_v2c`, gm120k, **vocab 200000** (larger embedding/unembedding than the 131k panel \u2014 fair size-match is o200k, not the other candidates), 124 specials, post-proc. Source `\u2026_consv2_eusino_v2c_gm120k_v200k_sp124_eng5g`. Std-1B 2571415+, mathcode 2571480+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60733582512, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7253267169497185, "blimp_acc": 0.8195373134328359, "code_bpb_mean": 0.5321604019509747, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9792521465200528, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.072, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.039423805913570885, "flores_mean_bpb_common": 2.9792521465200545, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 frde2 (more French/German, less Chinese) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_euh' (EU-dense, least fair of the four); roster #23; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130993058832, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34326210358173215, "code_bpb_mean": 0.3841518893095776, "flores_mean_bpb": 3.5365349823121583, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.182, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19787717968157695, "gsm8k_exact_match_flexible": 0.22896133434420016, "flores_mean_bpb_common": 3.536534982312159, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#23, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k`. **Apertus-v2 candidate** `preliminary_euh` (EU-dense Fr/De-boosted, Chinese cut; least fair of the four). consv2 `frde2` grouped config, gm110k, vocab 131072 + 124 specials, post-proc. Source `\u2026_consv2_frde2_gm110k_v131k_sp124_eng5g`. Std-1B 2571395+, mathcode 2571444+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 63678628703, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7277498476114497, "blimp_acc": 0.8207761194029849, "code_bpb_mean": 0.5317336699347034, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.968980373205703, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.088, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04169825625473844, "flores_mean_bpb_common": 2.9689803732057034, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex + repcap8 (8+ char run cap); fineweb2full, consv2 baseline (reparam ratios) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,017 (55-token ParityBpe shortfall vs 131,072 target); shipped Apertus-v2 candidate 'preliminary_mul' (balanced baseline, fairest of the four); roster #24; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137345073533, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34396997225653375, "code_bpb_mean": 0.3859327254314154, "flores_mean_bpb": 3.528042232456428, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.218, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17968157695223655, "gsm8k_exact_match_flexible": 0.21304018195602728, "flores_mean_bpb_common": 3.528042232456427, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#24, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8`. **Apertus-v2 candidate** `preliminary_mul` (balanced consv2 baseline, fairest of the four, no EU boost). `plus3`+`repcap8` consv2, vocab **131017** (55-token ParityBpe shortfall), no post-processor. Functionally distinct from roster #20 repcap8 (verified). Source `nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_v131k_sp124_eng5g`. Std-1B 2571405+, mathcode 2571462+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 63946138296, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7277884917270039, "blimp_acc": 0.8144328358208955, "code_bpb_mean": 0.5309802681278109, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.965308043536212, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.088, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.037149355572403335, "flores_mean_bpb_common": 2.965308043536213, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 + repcap8; fineweb2full, consv2 reparam (preliminary_mul) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072; the preliminary_mul build registered under an explicit slug (distinct md5 from the roster #24 twin); USED - S6h code analysis is not applicable (20B-only); Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137922050856, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3439376331489919, "code_bpb_mean": 0.386311590805194, "flores_mean_bpb": 3.504792999457629, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.212, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1865049279757392, "gsm8k_exact_match_flexible": 0.2259287338893101, "flores_mean_bpb_common": 3.5047929994576306, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#37, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072`. PA-BPE Apertus-v2 `preliminary_mul` variant, consv2, vocab 131,072 (md5 ead50f20). Distinct tokenizer from roster #24 (`\u2026-consv2-plus3-repcap8`, vocab 131,017, md5 879fd884). FINISHED (step 19073). MBPP 0.170 [0.138, 0.204].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 63127641060, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7277506388972832, "blimp_acc": 0.8183880597014924, "code_bpb_mean": 0.5288339782707951, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.970099624474484, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.072, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.028051554207733132, "gsm8k_exact_match_flexible": 0.039423805913570885, "flores_mean_bpb_common": 2.9700996244744844, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm=64000,W=100,a=2.0) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; hybrid-window pulls val_bpb/BLiMP/code_bpb back toward BPE-like at the cost of most of the all-FLORES gain; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61811355140, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7294609874470572, "blimp_acc": 0.819074626865672, "code_bpb_mean": 0.5337855917608801, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.952720239196703, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.086, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.034874905231235785, "flores_mean_bpb_common": 2.9527202391967036, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus2 regex; fineweb2full, capped + tuned (v5) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus2 tuned-data baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 61636322286, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7264930536694052, "blimp_acc": 0.8192985074626866, "code_bpb_mean": 0.5280546913441093, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9887710137252537, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.086, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.988771013725256, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus2 regex; fineweb2full, consv2 + tailcuts (6 families demoted to ratio 1.0) + eng5g (+5GB English) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (~131k + full Apertus chat-template specials, ~99 tokens); Apertus-2 development ablation 'PA-Clean-plus2-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132970204158, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34134989396481596, "code_bpb_mean": 0.3845164851887014, "flores_mean_bpb": 3.516050924775895, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.236, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18726307808946172, "gsm8k_exact_match_flexible": 0.21228203184230476, "flores_mean_bpb_common": 3.516050924775893, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#18, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g`. PA-BPE `plus2` + same `consv2-tailcuts-eng5g` data + specials config, global-merges=90k.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 134060493130, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34495804994861307, "code_bpb_mean": 0.38744267984589353, "flores_mean_bpb": 3.4562531366015037, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.198, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.22820318423047764, "flores_mean_bpb_common": 3.4562531366015032, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#14, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow`. PA-BPE with the `plus2` per-language vocab-allocation modifier. Finished 2026-06-03; mc_total 0.311 (highest in panel).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 63071957452, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7294688936122221, "blimp_acc": 0.8239552238805968, "code_bpb_mean": 0.5336782775861199, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.979246304587864, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.078, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.017437452615617893, "gsm8k_exact_match_flexible": 0.024260803639120546, "flores_mean_bpb_common": 2.979246304587864, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex; fineweb2full, consv2 (no tailcuts) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus3 consv2 baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 136794569534, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34411453574874007, "code_bpb_mean": 0.387961060061986, "flores_mean_bpb": 3.51622118667608, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.22, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19332827899924185, "gsm8k_exact_match_flexible": 0.221379833206975, "flores_mean_bpb_common": 3.5162211866760815, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#15, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2`. PA-BPE `plus3` + `consv2` data config. Finished 2026-06-05; mc_total 0.271 [0.249, 0.294] **below plus2's 0.311 [0.288, 0.335]** (mc_math subset Wilson CIs separate: 0.280 vs 0.354). MBPP / HumanEval / GSM8K all CI-overlap with plus2 \u2014 plus2 stays the better-performing variant.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 63690802707, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7275032359038739, "blimp_acc": 0.8209402985074628, "code_bpb_mean": 0.5291931590166571, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9754391952607704, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.09, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04473085670962851, "flores_mean_bpb_common": 2.97543919526077, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm70k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A6'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137402406971, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34293975437865554, "code_bpb_mean": 0.3820065458783474, "flores_mean_bpb": 3.500058100497096, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.214, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20090978013646701, "gsm8k_exact_match_flexible": 0.22971948445792267, "flores_mean_bpb_common": 3.500058100497093, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#17, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g`. PA-BPE `plus3` + `consv2-tailcuts-eng5g` data config (parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json), global-merges=70k, vocab 130929 with full Apertus chat-template specials (99 added tokens) baked in upstream.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 61617057747, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.726475413557967, "blimp_acc": 0.8127164179104476, "code_bpb_mean": 0.5286779776134367, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9638969184385138, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.098, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.034874905231235785, "gsm8k_exact_match_flexible": 0.039423805913570885, "flores_mean_bpb_common": 2.9638969184385147, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132928644091, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34171313250169755, "code_bpb_mean": 0.38502821035325424, "flores_mean_bpb": 3.5478004395829843, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.24, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.24336618650492797, "flores_mean_bpb_common": 3.547800439582986, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#19, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g`. PA-BPE `plus3` + same config as #18 but `plus3` pretok, global-merges=90k.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8": { "run": "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61615433001, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7262090143619204, "blimp_acc": 0.8244328358208957, "code_bpb_mean": 0.5266014430572129, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.008393821448739, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.098, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.0356330553449583, "flores_mean_bpb_common": 3.0083938214487427, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex + repcap8; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,141 (131,017 + 124 specials; Apertus precedent for >128k vocab); Apertus-2 development ablation 'PA-Clean-plus3-repcap8fr-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132895075611, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34175041096316866, "code_bpb_mean": 0.3855599711969472, "flores_mean_bpb": 3.5627034742441057, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.192, "humaneval_pass1": 0.18292682926829268, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2001516300227445, "gsm8k_exact_match_flexible": 0.22971948445792267, "flores_mean_bpb_common": 3.562703474244104, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#20, clean pretok, slug `pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8`. PA-BPE plus3+consv2+tailcuts+eng5g+gm90k+repcap8+fixed-regex variant. Vocab 131,017 + 124 specials (chat/reasoning/reserve slots); apertus precedent for >128k vocab. Source `nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr`. Distinct from roster #19 (`...gm90k-v130-sp-eng5g`) by md5 (5fafe099 vs 081e4ec3): differs by repcap8 + fixed-regex pretok. FINISHED step 19073 (resumed after an InvalidQOS stall at step 16000).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced": { "run": "full-128k-pabpe-nfc-gpt4-balanced", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 76284810542, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.7254768948770942, "blimp_acc": 0.816761194029851, "code_bpb_mean": 0.5335020022769154, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6645424496960075, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.029567854435178165, "flores_mean_bpb_common": 2.6645424496960066, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, grouped=25 script-family units) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (ParityBpeTrainer); vocab 127,826; bug-fixed 2026-05-16 re-export of the retired pabpe-balanced (2a); replaces the newline-stripping-bug original; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced-bytematched": { "run": "full-128k-pabpe-nfc-gpt4-balanced-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 10293, "total_batch_size": 1048576, "token_budget": 10792992768, "total_bytes_consumed": 89288100399, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.719335277931161, "flores_mean_bpb": 1.1570331965345826, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1570331965345826, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-gpt4-balanced-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-gpt4-balanced-bytematched_blimp_code_bpb.json; no full-128k-pabpe-nfc-gpt4-balanced-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-pabpe-nfc-gpt4-balanced-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-pabpe-nfc-gpt4-balanced-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-pabpe-nfc-gpt4-balanced-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow": { "run": "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 69596058966, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.7193782291843958, "blimp_acc": 0.816134328358209, "code_bpb_mean": 0.5239938767725334, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6586637109469846, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.06, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.0401819560272934, "flores_mean_bpb_common": 2.6586637109469846, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (global_merges=64000, window=100, alpha=2.0, signal=ratios) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,826; tests whether the hybrid-window scheme improves over base PA-BPE at fixed data/pretok; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch": { "run": "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 165451465939, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.3468567492776554, "code_bpb_mean": 0.38777905081348607, "flores_mean_bpb": 3.3580068430546723, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.13, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20318423047763456, "gsm8k_exact_match_flexible": 0.23805913570887036, "flores_mean_bpb_common": 3.3608354658623503, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#56, gpt4 pretok, slug `pabpe-nfc-gpt4-balanced`. Main-grid PA-BPE cell (first PA-BPE run at 20B). FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-fineweb2full": { "run": "full-128k-pabpe-nfc-gpt4-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127825, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72845529906, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127821, "metrics": { "val_bpb": 0.7447323227163575, "blimp_acc": 0.8236119402985074, "code_bpb_mean": 0.5606503370154372, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.8330004641502318, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.068, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.833000464150231, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow": { "run": "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127825, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 62964864120, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127821, "metrics": { "val_bpb": 0.7277658585120369, "blimp_acc": 0.827, "code_bpb_mean": 0.5309002808854102, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9691645570017604, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.074, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04852160727824109, "flores_mean_bpb_common": 2.9691645570017617, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-allmulti-equal-bpe": { "run": "full-128k-punct-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 61181692000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7222909830438362, "blimp_acc": 0.8184626865671643, "code_bpb_mean": 0.5298100623186873, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6519905802403945, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.651990580240394, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-punct-allmulti-equal-bpe_code_v2.json", "humaneval_pass1": "no full-128k-punct-allmulti-equal-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-punct-allmulti-equal-bpe_gsm8k_v2.json" }, "registry_note": "P1 Punctuation+ByteLevel; allmulti_equal data; norm=none; BPE; vocab 128,256; pairs with punct-balanced-bpe; USED - S6d.4 (rho=-0.375, weakest of the four - punct also compresses the tail least)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe": { "run": "full-128k-punct-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70269874400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7170390552723569, "blimp_acc": 0.8140149253731344, "code_bpb_mean": 0.5272228231489715, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.647426551753968, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.094, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03411675511751327, "flores_mean_bpb_common": 2.647426551753966, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P1 Punctuation+ByteLevel; balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3404114563994627, "blimp_acc": 0.5531044776119404, "code_bpb_mean": 0.38008554959295715, "blimp_scoring": null, "flores_mean_bpb": 3.332889085550648, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.23, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.25018953752843065, "flores_mean_bpb_common": 3.332889085550645, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "#13, punct pretok, slug `punct-balanced-bpe`. Punctuation-pretok BPE.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed42": { "run": "full-128k-punct-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70269874400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7185115767080312, "blimp_acc": 0.8170746268656716, "code_bpb_mean": 0.5302227287291184, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6495221045351793, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.076, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096, "flores_mean_bpb_common": 2.6495221045351793, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed42-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed42-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34034366356012596, "flores_mean_bpb": 3.3227860170269508, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.192, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 3.3250565916656507, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_code_bpb.json", "gsm8k_exact_match_strict": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed42-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34137498932537774, "flores_mean_bpb": 3.356340922797924, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.008, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18953752843062927, "gsm8k_exact_match_flexible": 0.2304776345716452, "flores_mean_bpb_common": 3.358734360607338, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3403527543062969, "flores_mean_bpb": 3.3464235720721844, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.084, "humaneval_pass1": 0.18292682926829268, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20394238059135708, "gsm8k_exact_match_flexible": 0.24791508718726307, "flores_mean_bpb_common": 3.348597720129644, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balancedhalf-superbpe-t64k-copy": { "run": "full-128k-punct-balancedhalf-superbpe-t64k-copy", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 68386628826, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balancedhalf-superbpe-t64k-copy", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7258874528699852, "blimp_acc": 0.807865671641791, "code_bpb_mean": 0.5273893624392161, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.635090338496232, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.044, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.029567854435178165, "flores_mean_bpb_common": 2.6350903384962305, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 Punct(Isolated)+ByteLevel base (64k transition) / stage-2 minimal regex (allows superwords to span punctuation); balanced-half (D2 halved) data; norm=none (inherited from punct base); SuperBPE; vocab 128,004; the only SuperBPE variant EXEMPT from the S9.5 combining-mark bug (its stage-2 regex has no [^\\s\\p{L}\\p{N}] alternative); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balancedhalf-superbpe-t64k-copy", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe": { "run": "full-128k-punct-english-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65285475200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7342410854286585, "blimp_acc": 0.8088805970149258, "code_bpb_mean": 0.5581842356992628, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.637858622428686, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.056, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.018953752843062926, "gsm8k_exact_match_flexible": 0.028051554207733132, "flores_mean_bpb_common": 2.6378586224286873, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P1 Punctuation+ByteLevel; english (D1) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe-mathcode-scratch": { "run": "full-128k-punct-english-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141498848692, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3434937319193573, "code_bpb_mean": 0.37235060078538484, "flores_mean_bpb": 3.1770676728331426, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.228, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20849128127369218, "gsm8k_exact_match_flexible": 0.22365428354814254, "flores_mean_bpb_common": 3.1818586879426354, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-punct-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#49, punct pretok, slug `punct-english-bpe`. Main-grid English-data BPE cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe-seed42": { "run": "full-128k-punct-english-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65285475200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7359940385712558, "blimp_acc": 0.8072686567164181, "code_bpb_mean": 0.5550694968769654, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.64065565785868, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.05, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04397270659590599, "flores_mean_bpb_common": 2.6406556578586793, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe": { "run": "full-128k-rightalign-balanced-bpe", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117764000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7120323882905231, "blimp_acc": 0.815567164179104, "code_bpb_mean": 0.5194966805778097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1394581789645044, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.082, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04094010614101592, "gsm8k_exact_match_flexible": 0.047763457164518575, "flores_mean_bpb_common": 1.1394581789645044, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits (REGEX_GPT4O_RIGHTALIGN); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe-continue": { "run": "full-128k-rightalign-balanced-bpe-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 73814886080, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6979263507221571, "blimp_acc": 0.6025223880597017, "code_bpb_mean": 0.5082600105542662, "blimp_scoring": null, "flores_mean_bpb": 1.118694277639942, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.086, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0576194086429113, "gsm8k_exact_match_flexible": 0.06141015921152388, "flores_mean_bpb_common": 1.1186942776399422, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe-mathcode-scratch": { "run": "full-128k-rightalign-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147637512815, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3352981899578387, "code_bpb_mean": 0.379809350915327, "flores_mean_bpb": 3.383188683529629, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.072, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23351023502653526, "gsm8k_exact_match_flexible": 0.265352539802881, "flores_mean_bpb_common": 3.385167825091684, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#52, rightalign pretok, slug `rightalign-balanced-bpe`. Main-grid RightAlign balanced BPE cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe-seed42": { "run": "full-128k-rightalign-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117764000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137766875032537, "blimp_acc": 0.8131492537313432, "code_bpb_mean": 0.5223377672129773, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.645338490029167, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.04624715693707354, "flores_mean_bpb_common": 2.6453384900291677, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe": { "run": "full-128k-rightalign-balanced-nfc-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68119057600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7119887597970906, "blimp_acc": 0.8136567164179105, "code_bpb_mean": 0.5193163415702758, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6383702373254945, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04397270659590599, "flores_mean_bpb_common": 2.6383702373254923, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147640316546, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3356379974663634, "code_bpb_mean": 0.38086114026234213, "flores_mean_bpb": 3.377811214706142, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.104, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22365428354814254, "gsm8k_exact_match_flexible": 0.26156178923426837, "flores_mean_bpb_common": 3.377811214706142, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#3, rightalign pretok, slug `rightalign-balanced-nfc-bpe`. Digit-direction matched pair vs (2).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe-seed42": { "run": "full-128k-rightalign-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68119057600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7138028167765688, "blimp_acc": 0.8157462686567161, "code_bpb_mean": 0.522530388505992, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.645945197075857, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.082, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.050037907505686124, "flores_mean_bpb_common": 2.645945197075857, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram": { "run": "full-128k-rightalign-balanced-unigram", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59928184800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7310138736467131, "blimp_acc": 0.8291940298507464, "code_bpb_mean": 0.5538140300742438, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6891200256408188, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.048, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.043214556482183475, "flores_mean_bpb_common": 2.689120025640816, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram-mathcode-scratch": { "run": "full-128k-rightalign-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129887530533, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.35354189301466415, "code_bpb_mean": 0.39891069834510856, "flores_mean_bpb": 3.403116477153282, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.0, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17513267626990145, "gsm8k_exact_match_flexible": 0.22971948445792267, "flores_mean_bpb_common": 3.4055218356876673, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#55, rightalign pretok, slug `rightalign-balanced-unigram`. Main-grid UnigramLM cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram-seed42": { "run": "full-128k-rightalign-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59928184800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7337618761598241, "blimp_acc": 0.8277164179104478, "code_bpb_mean": 0.5566215811979148, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.689153295734608, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.044, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.03411675511751327, "flores_mean_bpb_common": 2.689153295734608, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-bpe-scriptenc_gpt4o_cb": { "run": "full-128k-scripttok-bpe-scriptenc_gpt4o_cb", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 67708388000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-bpe-scriptenc_gpt4o_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7128471807824242, "blimp_acc": 0.8218507462686566, "code_bpb_mean": 0.5185598065707061, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.663102913833259, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.084, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04852160727824109, "gsm8k_exact_match_flexible": 0.05079605761940864, "flores_mean_bpb_common": 2.663102913833261, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "script_tok BPE, scriptenc_gpt4o_cb SCRIPT-encoding pretok; balanced (D2, script_tok's own fit sampler) data; norm=SCRIPT-encoding normalization; BPE (script_tok); vocab 128,256; requires pre-tokenized .npz shards; USED - panels.json structure-pretok (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-bpe-scriptenc_gpt4o_cb", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-continue": { "run": "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 73371271360, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "scripttok-bpe-scriptenc_gpt4o_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7005929724431691, "blimp_acc": 0.8206417910447761, "code_bpb_mean": 0.5113489581420481, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1225644132460593, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.102, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.05989385898407885, "gsm8k_exact_match_flexible": 0.06823351023502654, "flores_mean_bpb_common": 1.1225644132460595, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-bpe-scriptenc_gpt4o_cb-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch": { "run": "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 146750236855, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-bpe-scriptenc_gpt4o_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.33612096563369104, "code_bpb_mean": 0.37812117138811496, "flores_mean_bpb": 3.38748131750607, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.136, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2047005307050796, "gsm8k_exact_match_flexible": 0.23730098559514784, "flores_mean_bpb_common": 3.3899216335273605, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb": { "run": "full-128k-scripttok-mingram-scriptenc_cb", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70630463200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7124263503328025, "blimp_acc": 0.8125074626865672, "code_bpb_mean": 0.5125317350292097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.687708805401242, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.084, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04473085670962851, "gsm8k_exact_match_flexible": 0.04927975739196361, "flores_mean_bpb_common": 2.6877088054012432, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "script_tok MinGram, scriptenc_cb SCRIPT-encoding pretok (-n 126339), PUA/unassigned code points stripped; balanced (D2, script_tok's own fit sampler/composition/seed) data; norm=SCRIPT-encoding normalization (designed, applied identically train/eval); MinGram; vocab 128,256 (1,916 atomic + 126,339 learned + synthesized BOS); requires pre-tokenized .npz shards; forward-fusing (newline+following indentation fuse into one atomic pretoken, later diagnosed as the MBPP/HumanEval-collapse mechanism at 20B, S3a.1/3a.6/S6h); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb-continue": { "run": "full-128k-scripttok-mingram-scriptenc_cb-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 76537738304, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.6987038140560075, "blimp_acc": 0.8092835820895521, "code_bpb_mean": 0.5013382955836408, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.1185854280485894, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.116, "humaneval_pass1": 0.10365853658536585, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0712661106899166, "gsm8k_exact_match_flexible": 0.07278241091736164, "flores_mean_bpb_common": 1.1185854280485894, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch": { "run": "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153083502797, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.33485716807277754, "code_bpb_mean": 0.37927454694833107, "flores_mean_bpb": 3.4036921894353944, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.016, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19939347990902198, "gsm8k_exact_match_flexible": 0.24260803639120546, "flores_mean_bpb_common": 3.4063255605942415, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#57, mingram pretok, slug `scripttok-mingram-scriptenc_cb`. MinGram/SCRIPT-encoding fusing variant, the twin roster #48 (`scriptenc_cb_nl`) compares against. Row added retroactively 2026-08-10 (documentation gap: run predates #48). FINISHED; v2 evals: MBPP 0.016, HumanEval 0.1768, GSM8K 0.1994, code BPB 0.3793. Correlation-panel member; joined tab:mc20b-results as the 17th custom row 2026-08-10 (user full-coherence ruling, superseding an earlier off-roster classification).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb_nl": { "run": "full-128k-scripttok-mingram-scriptenc_cb_nl", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70578596000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb_nl", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7134216868447818, "blimp_acc": 0.8145074626865673, "code_bpb_mean": 0.5161016806694816, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6761030257693226, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "flores_mean_bpb_common": 2.676103025769323, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "mbpp_pass1": "no full-128k-scripttok-mingram-scriptenc_cb_nl_code_v2.json", "humaneval_pass1": "no full-128k-scripttok-mingram-scriptenc_cb_nl_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-scripttok-mingram-scriptenc_cb_nl_gsm8k_v2.json" }, "registry_note": "script_tok MinGram, scriptenc_cb_nl (split_line_breaks=True: newline isolated from indentation, additive opt-in); balanced (D2, balanced_tlm10g_cb_nl corpus); SCRIPT-encoding normalization; MinGram; vocab 128,256 (1,916 atomic+126,339 learned+BOS); line-break-splitting ablation vs scripttok-mingram-scriptenc_cb (newline-fusion mechanism). USED - S3a.6/S3a.7/S6h newline-fusion analysis (std-1B twin of 20B pair, finished 2026-08-08); design-variant ablation, not a panels.json member.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb_nl", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch": { "run": "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152971086535, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb_nl", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.336002609619353, "code_bpb_mean": 0.38298362754325693, "flores_mean_bpb": 3.4486723016230436, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.216, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2350265352539803, "gsm8k_exact_match_flexible": 0.2721758908263836, "flores_mean_bpb_common": 3.4508243876359357, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#48, mingram pretok, slug `scripttok-mingram-scriptenc_cb_nl`. MinGram/SCRIPT-encoding line-break-splitting variant (`split_line_breaks`, opt-in additive), the non-GPT4o-family test of the same mechanism. FINISHED. **MBPP 0.008 \u2192 0.214** vs its fusing twin (roster row for `scriptenc_cb`), HumanEval 0.000 \u2192 0.0610, GSM8K 0.216 \u2192 0.244. Pre-registered prediction (MBPP into 0.148-0.220) CONFIRMED. Evals under the 2780479-2780482 chain. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow": { "run": "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 58123160129, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7330722568700316, "blimp_acc": 0.8146716417910445, "code_bpb_mean": 0.5405010740637953, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.0808713247974078, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.074, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.03639120545868082, "flores_mean_bpb_common": 3.080871324797409, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "SuperBPE on the apertus-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate; USED - Apertus-2 development track candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 126018305461, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34260583858556143, "code_bpb_mean": 0.3836187126285858, "flores_mean_bpb": 3.663785438014377, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.012, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19257012888551933, "gsm8k_exact_match_flexible": 0.22744503411675512, "flores_mean_bpb_common": 3.6637854380143797, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#11, apertus pretok, slug `superbpe-apertus-fineweb2full-capped-hybridwindow`. SuperBPE candidate on apertus pretok.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c2-balanced": { "run": "full-128k-superbpe-clean-c2-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 72553591786, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-c2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7285011810271208, "blimp_acc": 0.8114328358208953, "code_bpb_mean": 0.5259040594970449, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6527426081302217, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.084, "humaneval_pass1": 0.11585365853658537, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03639120545868082, "flores_mean_bpb_common": 2.6527426081302243, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 clean-multi (PA-BPE clean base) / stage-2 clean_multi C2 reduced regex; balanced (D2) data; norm=NFC (inherited from PA-BPE clean base); SuperBPE; vocab 128,004; Phase-5 clean_multi_c2_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c2-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c2-balanced-mathcode-scratch": { "run": "full-128k-superbpe-clean-c2-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 157305292274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-c2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34319379203223876, "code_bpb_mean": 0.39056603890195457, "flores_mean_bpb": 3.351038195492764, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.202, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.24109173616376042, "gsm8k_exact_match_flexible": 0.244882486732373, "flores_mean_bpb_common": 3.3510381954927637, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-c2-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#33, clean pretok, slug `superbpe-clean-c2-balanced`. SuperBPE, clean_multi C2 stage-2 reduced regex on the PA-BPE clean base, `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.176 [0.144, 0.210].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c2-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c3-balanced": { "run": "full-128k-superbpe-clean-c3-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70479910164, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-c3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7295809612395211, "blimp_acc": 0.8030746268656719, "code_bpb_mean": 0.5305765831960391, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.647679513788064, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.044, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.03184230477634572, "flores_mean_bpb_common": 2.647679513788064, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 clean-multi / stage-2 clean_multi C3 reduced regex; balanced (D2) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 clean_multi_c3_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c3-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c3-balanced-mathcode-scratch": { "run": "full-128k-superbpe-clean-c3-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152809290276, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-c3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.344799716331132, "code_bpb_mean": 0.38962529072156843, "flores_mean_bpb": 3.3606069071813196, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.19, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17058377558756635, "gsm8k_exact_match_flexible": 0.20242608036391205, "flores_mean_bpb_common": 3.36060690718132, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-c3-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#34, clean pretok, slug `superbpe-clean-c3-balanced`. SuperBPE, clean_multi C3 stage-2 reduced regex, `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.200 [0.166, 0.236].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c3-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 59036341511, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7322630359017345, "blimp_acc": 0.8139253731343283, "code_bpb_mean": 0.5362691238736629, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.038298249585932, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.09, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.0356330553449583, "flores_mean_bpb_common": 3.0382982495859334, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "SuperBPE on the clean-multi-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate (t90k/v128k); USED - Apertus-2 development track candidate + S6h code analysis roster (20B twin only, see MC table); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 127998197299, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.3423861965022762, "code_bpb_mean": 0.3918066963394801, "flores_mean_bpb": 3.640373138824759, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.208, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2047005307050796, "gsm8k_exact_match_flexible": 0.2175890826383624, "flores_mean_bpb_common": 3.640373138824758, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#9, clean pretok, slug `superbpe-clean-fineweb2full-capped-hybridwindow`. SuperBPE candidate (clean, t90k/v128k).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 8827, "total_batch_size": 1048576, "token_budget": 9255780352, "total_bytes_consumed": 60452486276, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.7318322821516513, "blimp_acc": 0.8206417910447763, "code_bpb_mean": 0.5338679088662601, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9932878985914204, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.106, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.039423805913570885, "flores_mean_bpb_common": 2.993287898591421, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "SuperBPE, clean-multi pretok base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab ~130,000; SuperBPE stage-2 transition-point ablation (t110k vs the 2x2 candidate's t90k); USED - Apertus-2 development track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130623118924, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.34328670524163585, "code_bpb_mean": 0.39020128491208583, "flores_mean_bpb": 3.62831755593602, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.218, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17892342683851403, "gsm8k_exact_match_flexible": 0.2100075815011372, "flores_mean_bpb_common": 3.6283175559360186, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#10, clean pretok, slug `superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k`. SuperBPE ablation (t110k/v130k transition point).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2": { "run": "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 8827, "total_batch_size": 1048576, "token_budget": 9255780352, "total_bytes_consumed": 61543706497, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.7322648630229084, "blimp_acc": 0.8177313432835819, "code_bpb_mean": 0.5352055661202627, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.0648179432560387, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.084, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.043214556482183475, "flores_mean_bpb_common": 3.06481794325604, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": {}, "registry_note": "SuperBPE on the clean-multi plus2 (v2) capped-v2 hybrid-window base, stage-2 transition t110k; fineweb2full, consv2 data; norm=NFC; SuperBPE; vocab ~130,000; Apertus-2 development ablation 'SuperBPE-plus2v2-cv2-t110k'; USED - Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132980980403, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.34330320732271896, "code_bpb_mean": 0.38956319484965596, "flores_mean_bpb": 3.559493631071275, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.246, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.16603487490523122, "gsm8k_exact_match_flexible": 0.20166793025018953, "flores_mean_bpb_common": 3.559493631071274, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#16, clean pretok, slug `superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2`. SuperBPE on clean-multi + `plus2` + `cap_v2` + hw + `consv2` data, stage-2 transition 110k \u2192 130k. Specials added post-hoc (vocab 130000 \u2192 130004, bos_id=130000).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 126294692304, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-gpt4-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34104167417539477, "code_bpb_mean": 0.38122511963402833, "flores_mean_bpb": 3.6204409397662, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.162, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17134192570128887, "gsm8k_exact_match_flexible": 0.20773313115996966, "flores_mean_bpb_common": 3.6204409397662016, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#12, gpt4 pretok, slug `superbpe-gpt4-fineweb2full-hybridwindow`. SuperBPE on older GPT-4 regex (uncapped, {1,3}-digit).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced": { "run": "full-128k-superbpe-gpt4o-balanced", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70221489492, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7236170040912355, "blimp_acc": 0.8035373134328355, "code_bpb_mean": 0.5249302535411448, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644030384628379, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.052, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03335860500379075, "gsm8k_exact_match_flexible": 0.03411675511751327, "flores_mean_bpb_common": 2.6440303846283797, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex (90k inherited merges) / stage-2 reduced GPT-4o regex (38k superword merges); base = gpt4o-balanced-bpe; balanced (D2, halved for stage-2: balanced_half.json) data; norm=none (inherited from gpt4o-balanced-bpe base); SuperBPE; vocab 128,004; Phase-5 fixed retrain (2026-05-18) of the combining-mark-bug Run 1; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-bytematched": { "run": "full-128k-superbpe-gpt4o-balanced-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 7651, "total_batch_size": 1048576, "token_budget": 8022654976, "total_bytes_consumed": 61073617836, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7255596892054963, "flores_mean_bpb": 1.1545314705147296, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1545314705147296, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-gpt4o-balanced-bytematched_blimp_code_bpb.json; no full-128k-superbpe-gpt4o-balanced-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-superbpe-gpt4o-balanced-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-superbpe-gpt4o-balanced-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-continue": { "run": "full-128k-superbpe-gpt4o-balanced-continue", "works": [ "paper_a" ], "training_regime": "continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 76120509696, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-continue" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.708866739434701, "flores_mean_bpb": 1.1350611921815752, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1350611921815752, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-continue_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-gpt4o-balanced-continue_blimp_code_bpb.json; no full-128k-superbpe-gpt4o-balanced-continue_code_bpb.json", "mbpp_pass1": "no full-128k-superbpe-gpt4o-balanced-continue_code_v2.json", "humaneval_pass1": "no full-128k-superbpe-gpt4o-balanced-continue_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-continue_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-continue", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-fresh80": { "run": "full-128k-superbpe-gpt4o-balanced-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70221489492, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7186237343159342, "flores_mean_bpb": 1.1322591385679686, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1322591385679683, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-gpt4o-balanced-fresh80_blimp_code_bpb.json; no full-128k-superbpe-gpt4o-balanced-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-superbpe-gpt4o-balanced-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-superbpe-gpt4o-balanced-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160": { "run": "full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 76120509696, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.6886954840735489, "flores_mean_bpb": 1.1032346789130267, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1032346789130267, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-mathcode-scratch": { "run": "full-128k-superbpe-gpt4o-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152249001828, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.33859987485722765, "code_bpb_mean": 0.38337076287095095, "flores_mean_bpb": 3.332138865321979, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.068, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18043972706595907, "gsm8k_exact_match_flexible": 0.2266868840030326, "flores_mean_bpb_common": 3.332138865321979, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#35, gpt4o pretok, slug `superbpe-gpt4o-balanced`. SuperBPE, gpt4o base (no norm, t90k), `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.106 [0.080, 0.134].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-seed43": { "run": "full-128k-superbpe-gpt4o-balanced-seed43", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70221489492, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7236979947317723, "blimp_acc": 0.8044925373134328, "code_bpb_mean": 0.523996973962109, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.645438208377306, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.074, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.6454382083773047, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-seed43_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-seed43", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-seed44": { "run": "full-128k-superbpe-gpt4o-balanced-seed44", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70221489492, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7242824771876463, "blimp_acc": 0.8094776119402984, "code_bpb_mean": 0.5254571888056621, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6418806386240528, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.058, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "flores_mean_bpb_common": 2.641880638624052, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": { "gsm8k_exact_match_strict": "no full-128k-superbpe-gpt4o-balanced-seed44_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-seed44", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-pabpe-nfc-gpt4-balanced": { "run": "full-128k-superbpe-pabpe-nfc-gpt4-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 71113918751, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.729206724155245, "blimp_acc": 0.8013283582089552, "code_bpb_mean": 0.5276386372741176, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6579343011899477, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.064, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03790750568612585, "flores_mean_bpb_common": 2.657934301189947, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC (inherited from PA-BPE base); SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 2; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched": { "run": "full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8159, "total_batch_size": 1048576, "token_budget": 8555331584, "total_bytes_consumed": 65956401397, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7307186210679458, "flores_mean_bpb": 1.1727217026790477, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1727217026790475, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_blimp_code_bpb.json; no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-pabpe-nfc-gpt4-balanced-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-pabpe-nfc-gpt4-t64k": { "run": "full-128k-superbpe-pabpe-nfc-gpt4-t64k", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 69736009453, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-pabpe-nfc-gpt4-t64k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7293487935520814, "blimp_acc": 0.7918208955223883, "code_bpb_mean": 0.529837856895328, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.658079093250394, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.046, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.026535253980288095, "gsm8k_exact_match_flexible": 0.028051554207733132, "flores_mean_bpb_common": 2.658079093250392, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex (64k transition, more aggressive superword fusing) / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 4 (t64k transition point vs t90k); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-pabpe-nfc-gpt4-t64k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe": { "run": "full-128k-whitespace-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 66097600800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.720734254582808, "blimp_acc": 0.8136268656716417, "code_bpb_mean": 0.5269486944981118, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6262572576348284, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.056, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04094010614101592, "flores_mean_bpb_common": 2.6262572576348284, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "WhitespaceSplit(preserve)+ByteLevel (minimal-structure anchor); balanced (D2) data; norm=none; BPE; vocab 128,260; lower-end pretok anchor; forward-fusing (whitespace+following word one pretoken); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe-bytematched": { "run": "full-128k-whitespace-balanced-bpe-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 7969, "total_batch_size": 1048576, "token_budget": 8356102144, "total_bytes_consumed": 59855884179, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7217743879771611, "flores_mean_bpb": 1.145415240386689, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1454152403866888, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-bpe-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-whitespace-balanced-bpe-bytematched_blimp_code_bpb.json; no full-128k-whitespace-balanced-bpe-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-whitespace-balanced-bpe-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-whitespace-balanced-bpe-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-whitespace-balanced-bpe-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe-fresh80": { "run": "full-128k-whitespace-balanced-bpe-fresh80", "works": [ "paper_a" ], "training_regime": "single-pass-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 66097600800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7168184984753029, "flores_mean_bpb": 1.1262626589258324, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1262626589258324, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-bpe-fresh80_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-whitespace-balanced-bpe-fresh80_blimp_code_bpb.json; no full-128k-whitespace-balanced-bpe-fresh80_code_bpb.json", "mbpp_pass1": "no full-128k-whitespace-balanced-bpe-fresh80_code_v2.json", "humaneval_pass1": "no full-128k-whitespace-balanced-bpe-fresh80_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-whitespace-balanced-bpe-fresh80_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe-fresh80", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160": { "run": "full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160", "works": [ "paper_a" ], "training_regime": "single-pass-continue-10B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 9536, "total_batch_size": 1048576, "token_budget": 9999220736, "total_bytes_consumed": 71625763776, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.6869904380660102, "flores_mean_bpb": 1.0963780029916184, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.0963780029916181, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_blimp_code_bpb.json; no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_code_bpb.json", "mbpp_pass1": "no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "humaneval_pass1": "no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe-fresh80-continue-fresh160", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe-mathcode-scratch": { "run": "full-128k-whitespace-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143259038643, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3395392358308889, "code_bpb_mean": 0.3742340349376568, "flores_mean_bpb": 3.321987419368069, "flores_n_languages": 214, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "mbpp_pass1": 0.102, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.177407126611069, "gsm8k_exact_match_flexible": 0.20621683093252463, "flores_mean_bpb_common": 3.321987419368068, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#36, whitespace pretok, slug `whitespace-balanced-bpe`. BPE with minimal WhitespaceSplit + ByteLevel pretok (lower-end pretok anchor), `balanced`. FINISHED (step 19073). MBPP 0.006 [0.000, 0.014].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-unigram": { "run": "full-128k-whitespace-balanced-unigram", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 73127700800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8400830729672958, "blimp_acc": 0.8051940298507458, "code_bpb_mean": 0.6091013373748696, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.577697347573601, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.04, "humaneval_pass1": 0.0, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": false, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": false, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.032600454890068235, "flores_mean_bpb_common": 2.577697347573601, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "Metaspace(replacement=U+2581)+ByteFallback (SPM native mode); balanced (D2) data; norm=none; UnigramLM (Google sentencepiece, mode=native; tuned kwargs matched to UNIGRAM_TUNED_KWARGS); vocab 128,260; algorithm-axis Unigram counterpart to whitespace-balanced-bpe; trained via SPM not HF (HF UnigramTrainer NaN-panics on WhitespaceSplit+CJK); USED - S5e AXES-2 whitespace comparison + balanced-1B bootstrap panel (S6.12); not in panels.json groups", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-unigram-bytematched": { "run": "full-128k-whitespace-balanced-unigram-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 12482, "total_batch_size": 1048576, "token_budget": 13088325632, "total_bytes_consumed": 103724995612, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.828232917055742, "flores_mean_bpb": 1.1443914188391222, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1443914188391222, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-unigram-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-whitespace-balanced-unigram-bytematched_blimp_code_bpb.json; no full-128k-whitespace-balanced-unigram-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-whitespace-balanced-unigram-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-whitespace-balanced-unigram-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-whitespace-balanced-unigram-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-unigram-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-multiheavy-bpe": { "run": "full-128k-whitespace-multiheavy-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70848465600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-multiheavy-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7200585315424582, "blimp_acc": 0.8089552238805972, "code_bpb_mean": 0.5272055135520145, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6327296434017735, "flores_n_languages": 214, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "all214", "mbpp_pass1": 0.072, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02350265352539803, "gsm8k_exact_match_flexible": 0.030326004548900682, "flores_mean_bpb_common": 2.632729643401775, "flores_n_languages_common": 214, "flores_common_basis": "full1012/all214" }, "metric_gaps": {}, "registry_note": "WhitespaceSplit(preserve)+ByteLevel; multiheavy (30 FineWeb-2 lang weights doubled 0.30->0.60; english 0.175, math/code 0.0875 each) data; norm=none; BPE; vocab 128,256; data-compensation test: does doubling the multilingual budget offset whitespace-pretok's multilingual damage? (S5e: no, MBPP stays 0.002); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-multiheavy-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-multiheavy-bpe-bytematched": { "run": "full-128k-whitespace-multiheavy-bpe-bytematched", "works": [ "paper_a" ], "training_regime": "byte-matched-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8189, "total_batch_size": 1048576, "token_budget": 8586788864, "total_bytes_consumed": 65929327818, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-multiheavy-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7199543843753294, "flores_mean_bpb": 1.1442568180575319, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.1442568180575319, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-multiheavy-bpe-bytematched_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-whitespace-multiheavy-bpe-bytematched_blimp_code_bpb.json; no full-128k-whitespace-multiheavy-bpe-bytematched_code_bpb.json", "mbpp_pass1": "no full-128k-whitespace-multiheavy-bpe-bytematched_code_v2.json", "humaneval_pass1": "no full-128k-whitespace-multiheavy-bpe-bytematched_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-whitespace-multiheavy-bpe-bytematched_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-multiheavy-bpe-bytematched", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-apertus": { "run": "pilot-128k-apertus", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 131072, "total_parameters": 605029178 }, "training": { "step": 3370, "total_batch_size": 1048576, "token_budget": 3533701120, "total_bytes_consumed": 23542361680, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.8233875438120292, "flores_mean_bpb": 1.2772693692893287, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.2772693692893289, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no pilot-128k-apertus_blimp_code_bpb.json", "code_bpb_mean": "no pilot-128k-apertus_blimp_code_bpb.json; no pilot-128k-apertus_code_bpb.json", "mbpp_pass1": "no pilot-128k-apertus_code_v2.json", "humaneval_pass1": "no pilot-128k-apertus_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-apertus_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-apertus-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-apertus", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-apertus-wdfix": { "run": "pilot-128k-apertus-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 605029178 }, "training": { "step": 3370, "total_batch_size": 1048576, "token_budget": 3533701120, "total_bytes_consumed": 23542361680, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "apertus", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.8210034941193963, "blimp_acc": 0.7977014925373136, "code_bpb_mean": 0.6115311977869082, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.826757759606348, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.833011943999572, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-apertus-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-apertus-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-apertus-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-apertus`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-apertus`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-apertus-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-bpe": { "run": "pilot-128k-claude-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26503180769, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8130946827374702, "blimp_acc": 0.8036716417910448, "code_bpb_mean": 0.5993885155419614, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 1.2739596760972292, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.2739596760972292, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-nfc-bpe": { "run": "pilot-128k-claude-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26503127313, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8137996131939872, "blimp_acc": 0.6210746268656718, "code_bpb_mean": 0.6015235015799788, "blimp_scoring": null, "flores_mean_bpb": 1.272015204260806, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.272015204260806, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-unigram": { "run": "pilot-128k-claude-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22735755575, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8470803384113235, "blimp_acc": 0.5728656716417911, "code_bpb_mean": 0.6425699405251949, "blimp_scoring": null, "flores_mean_bpb": 1.3184939127433213, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3184939127433213, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-english-bpe": { "run": "pilot-128k-claude-english-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25009823930, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8326818042159945, "blimp_acc": 0.6018955223880598, "code_bpb_mean": 0.6386502722656455, "blimp_scoring": null, "flores_mean_bpb": 1.3284085906568848, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3284085906568845, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-bpe": { "run": "pilot-128k-gpt4o-balanced-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861307849, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8096293893125867, "flores_mean_bpb": 1.2744312240339906, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.2744312240339908, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no pilot-128k-gpt4o-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no pilot-128k-gpt4o-balanced-bpe_blimp_code_bpb.json; no pilot-128k-gpt4o-balanced-bpe_code_bpb.json", "mbpp_pass1": "no pilot-128k-gpt4o-balanced-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-gpt4o-balanced-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-bpe-wdfix": { "run": "pilot-128k-gpt4o-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861307849, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8066623035629523, "blimp_acc": 0.8082985074626864, "code_bpb_mean": 0.5994759350843143, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7343951601156684, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.7402135317772616, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-gpt4o-balanced-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-gpt4o-balanced-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-nfc-bpe": { "run": "pilot-128k-gpt4o-balanced-nfc-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861802317, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8096801983090485, "blimp_acc": 0.6228805970149255, "code_bpb_mean": 0.5992929454488137, "blimp_scoring": null, "flores_mean_bpb": 1.2721424267021788, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.272142426702179, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-gpt4o-balanced-nfc-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix": { "run": "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861802317, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8064211739874031, "blimp_acc": 0.8022089552238807, "code_bpb_mean": 0.5986240643292752, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7400664324472417, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.7459071779365103, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-gpt4o-balanced-nfc-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-gpt4o-balanced-nfc-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-unigram": { "run": "pilot-128k-gpt4o-balanced-unigram", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752717832, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8441244779079309, "blimp_acc": 0.5740597014925372, "code_bpb_mean": 0.643624308778441, "blimp_scoring": null, "flores_mean_bpb": 1.3143470089687153, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3143470089687153, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-unigram_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-unigram_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-gpt4o-balanced-unigram-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-unigram", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-unigram-wdfix": { "run": "pilot-128k-gpt4o-balanced-unigram-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752717832, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8407208082454437, "blimp_acc": 0.8078955223880595, "code_bpb_mean": 0.6417422373886053, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.78582454208114, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.791754861442065, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-unigram-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-unigram-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-unigram-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-gpt4o-balanced-unigram`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-gpt4o-balanced-unigram`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-unigram-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-code-bpe": { "run": "pilot-128k-gpt4o-code-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22652143709, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8300086036048998, "flores_mean_bpb": 1.3173980299350638, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3173980299350638, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no pilot-128k-gpt4o-code-bpe_blimp_code_bpb.json", "code_bpb_mean": "no pilot-128k-gpt4o-code-bpe_blimp_code_bpb.json; no pilot-128k-gpt4o-code-bpe_code_bpb.json", "mbpp_pass1": "no pilot-128k-gpt4o-code-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-code-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-code-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-gpt4o-code-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-code-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-code-bpe-wdfix": { "run": "pilot-128k-gpt4o-code-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22652143709, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8281893407285823, "blimp_acc": 0.8007014925373133, "code_bpb_mean": 0.6246381299311335, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7373735478233345, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.7431237374451674, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-code-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-code-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-code-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-gpt4o-code-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-gpt4o-code-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-code-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-english-bpe-wdfix": { "run": "pilot-128k-gpt4o-english-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 24726677521, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8326820796903395, "blimp_acc": 0.8165970149253732, "code_bpb_mean": 0.6484054540133121, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.749240218556993, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.755055780449086, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-english-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-english-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-english-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-english-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-llama3": { "run": "pilot-128k-llama3", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22614256769, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8215220013765772, "flores_mean_bpb": 1.2913061071163483, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.2913061071163483, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no pilot-128k-llama3_blimp_code_bpb.json", "code_bpb_mean": "no pilot-128k-llama3_blimp_code_bpb.json; no pilot-128k-llama3_code_bpb.json", "mbpp_pass1": "no pilot-128k-llama3_code_v2.json", "humaneval_pass1": "no pilot-128k-llama3_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-llama3_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-llama3-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-llama3", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-llama3-wdfix": { "run": "pilot-128k-llama3-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22614256769, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "llama3", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8192633301410291, "blimp_acc": 0.8085820895522388, "code_bpb_mean": 0.6220985225846258, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7514998067269696, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.7573763269549603, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-llama3-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-llama3-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-llama3-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-llama3`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-llama3`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-llama3-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-pabpe-nfc-gpt4-balanced": { "run": "pilot-128k-pabpe-nfc-gpt4-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 595198778 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.8333108785145202, "blimp_acc": 0.5588805970149254, "code_bpb_mean": 0.6240723541441774, "blimp_scoring": null, "flores_mean_bpb": 1.2790069641156285, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.016, "humaneval_pass1": 0.024390243902439025, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.011372251705837756, "gsm8k_exact_match_flexible": 0.013646702047005308, "flores_mean_bpb_common": 1.2790069641156288, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-balanced-bpe": { "run": "pilot-128k-punct-balanced-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26678596633, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8213861923632345, "flores_mean_bpb": 1.2831863426414045, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.283186342641404, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no pilot-128k-punct-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no pilot-128k-punct-balanced-bpe_blimp_code_bpb.json; no pilot-128k-punct-balanced-bpe_code_bpb.json", "mbpp_pass1": "no pilot-128k-punct-balanced-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-balanced-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-punct-balanced-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-balanced-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-balanced-bpe-wdfix": { "run": "pilot-128k-punct-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26678596633, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8178602531852399, "blimp_acc": 0.8050149253731345, "code_bpb_mean": 0.6131823552459643, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7370705187894253, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.7428741681621265, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-punct-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-punct-balanced-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-punct-balanced-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-english-bpe": { "run": "pilot-128k-punct-english-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 24786224164, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8356923683955118, "blimp_acc": 0.6069850746268659, "code_bpb_mean": 0.651206315313653, "blimp_scoring": null, "flores_mean_bpb": 1.3319003476516982, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.331900347651698, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-punct-english-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-english-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-punct-english-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-english-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-english-bpe-wdfix": { "run": "pilot-128k-punct-english-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 24786224164, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8333547427451476, "blimp_acc": 0.801268656716418, "code_bpb_mean": 0.6463363568675289, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7478260022807928, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.753625823733986, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-punct-english-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-english-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-english-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-punct-english-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-punct-english-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-english-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-bpe": { "run": "pilot-128k-rightalign-balanced-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861528355, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.809335826084626, "blimp_acc": 0.588313432835821, "code_bpb_mean": 0.5994605061825855, "blimp_scoring": null, "flores_mean_bpb": 1.2740973913252567, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.2740973913252562, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-rightalign-balanced-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-bpe-wdfix": { "run": "pilot-128k-rightalign-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861528355, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.806576174927823, "blimp_acc": 0.8065522388059699, "code_bpb_mean": 0.5993113828341335, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.735799278750796, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.741614350043495, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-rightalign-balanced-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-rightalign-balanced-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-nfc-bpe": { "run": "pilot-128k-rightalign-balanced-nfc-bpe", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25862019482, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8094262725329053, "blimp_acc": 0.6135074626865671, "code_bpb_mean": 0.5988623625501533, "blimp_scoring": null, "flores_mean_bpb": 1.2721536059858707, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.272153605985871, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-rightalign-balanced-nfc-bpe-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-nfc-bpe", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-nfc-bpe-wdfix": { "run": "pilot-128k-rightalign-balanced-nfc-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25862019482, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8064537576731613, "blimp_acc": 0.8079850746268656, "code_bpb_mean": 0.5993740119523097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7336603165642184, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.73951303746614, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-rightalign-balanced-nfc-bpe`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-rightalign-balanced-nfc-bpe`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-nfc-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-unigram": { "run": "pilot-128k-rightalign-balanced-unigram", "works": [ "paper_a_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752280161, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8448688676974012, "blimp_acc": 0.6195820895522387, "code_bpb_mean": 0.6427791454642018, "blimp_scoring": null, "flores_mean_bpb": 1.3160118856519614, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3160118856519614, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-unigram_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-unigram_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this run used the larger-than-intended weight decay documented in .claude/CLAUDE.md (effective 0.396 where 0.131 was intended). The checkpoint metadata records only the unscaled base value 0.28, because train.py scales it at run time (lambda = base * sqrt(B/B_REF) * (D_REF/D)), so this run's settings table is identical to its `-wdfix` sibling's and the two differ only in their measured values. The cross-scale ranking result of the cross-lingual paper is computed from THIS run, not from `pilot-128k-rightalign-balanced-unigram-wdfix`. Absolute performance is degraded; the comparison across tokenizers at this scale is not, because all d16 runs were affected equally.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-unigram", "repo_commit": "0a18016", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-unigram-wdfix": { "run": "pilot-128k-rightalign-balanced-unigram-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752280161, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.840835607389188, "blimp_acc": 0.8000149253731341, "code_bpb_mean": 0.6415251666607594, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.78969682999519, "flores_n_languages": 215, "flores_basis": "first200", "flores_sentences_per_language": 200, "flores_language_scope": null, "flores_mean_bpb_common": 2.795613630262447, "flores_n_languages_common": 214, "flores_common_basis": "first200/all" }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-unigram-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-unigram-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-unigram-wdfix_gsm8k_v2.json" }, "registry_note": "WEIGHT DECAY: this is the corrected rerun of `pilot-128k-rightalign-balanced-unigram`, which used the larger-than-intended weight decay documented in .claude/CLAUDE.md. Both runs record the same unscaled base value 0.28 in their metadata, so pick between them by name: the cross-lingual paper's cross-scale ranking result uses `pilot-128k-rightalign-balanced-unigram`, not this run.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-unigram-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-superbpe-gpt4o-balanced": { "run": "pilot-128k-superbpe-gpt4o-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 595788602 }, "training": { "step": 3339, "total_batch_size": 1048576, "token_budget": 3501195264, "total_bytes_consumed": 26653353804, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8097538729950767, "blimp_acc": 0.5948955223880596, "code_bpb_mean": 0.5919398272220795, "blimp_scoring": null, "flores_mean_bpb": 1.2780287302016062, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.014, "humaneval_pass1": 0.024390243902439025, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.01288855193328279, "gsm8k_exact_match_flexible": 0.01592115238817286, "flores_mean_bpb_common": 1.2780287302016065, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/pilot-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-apertus": { "run": "small-128k-apertus", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 387514850 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 23311828168, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.865119424983589, "flores_mean_bpb": 1.3321204840698462, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3321204840698462, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-apertus_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-apertus_blimp_code_bpb.json; no small-128k-apertus_code_bpb.json", "mbpp_pass1": "no small-128k-apertus_code_v2.json", "humaneval_pass1": "no small-128k-apertus_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-apertus_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-bpe": { "run": "small-128k-claude-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471449933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8528457701568503, "flores_mean_bpb": 1.3266824142659635, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3266824142659632, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-bpe_blimp_code_bpb.json; no small-128k-claude-balanced-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-nfc-bpe": { "run": "small-128k-claude-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471396541, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8523228950373269, "flores_mean_bpb": 1.3244926107942343, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3244926107942343, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-claude-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-unigram": { "run": "small-128k-claude-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22708535275, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.888320485067197, "flores_mean_bpb": 1.3699840024094312, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3699840024094312, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-unigram_blimp_code_bpb.json; no small-128k-claude-balanced-unigram_code_bpb.json", "mbpp_pass1": "no small-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-english-bpe": { "run": "small-128k-claude-english-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24979881010, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8739654201888126, "flores_mean_bpb": 1.4009236172250235, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4009236172250232, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-claude-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-english-bpe_blimp_code_bpb.json; no small-128k-claude-english-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-bpe": { "run": "small-128k-gpt4o-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830345493, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8488644192005518, "flores_mean_bpb": 1.3258177794440946, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3258177794440946, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-bpe_blimp_code_bpb.json; no small-128k-gpt4o-balanced-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-nfc-bpe": { "run": "small-128k-gpt4o-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830839369, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8486585324292858, "flores_mean_bpb": 1.3240801091460686, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3240801091460686, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-gpt4o-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-unigram": { "run": "small-128k-gpt4o-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725477224, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8836673905213913, "flores_mean_bpb": 1.3626724266864918, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3626724266864918, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-unigram_blimp_code_bpb.json; no small-128k-gpt4o-balanced-unigram_code_bpb.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-code-bpe": { "run": "small-128k-gpt4o-code-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22625023513, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8719229906379045, "flores_mean_bpb": 1.3862592127373865, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3862592127373865, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-code-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-code-bpe_blimp_code_bpb.json; no small-128k-gpt4o-code-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-gpt4o-code-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-code-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-code-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-english-bpe": { "run": "small-128k-gpt4o-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24697073597, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8781429007208184 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-english-bpe_blimp_code_bpb.json; no small-128k-gpt4o-english-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-english-bpe.json and no full-devtest result", "mbpp_pass1": "no small-128k-gpt4o-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-llama3": { "run": "small-128k-llama3", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22587181933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8639087471134697, "flores_mean_bpb": 1.3516025825109685, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3516025825109683, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-llama3_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-llama3_blimp_code_bpb.json; no small-128k-llama3_code_bpb.json", "mbpp_pass1": "no small-128k-llama3_code_v2.json", "humaneval_pass1": "no small-128k-llama3_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-llama3_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-pabpe-nfc-gpt4-balanced": { "run": "small-128k-pabpe-nfc-gpt4-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 380142050 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.8801238225129392, "blimp_acc": 0.5636865671641792, "code_bpb_mean": 0.666750653407198, "blimp_scoring": null, "flores_mean_bpb": 1.3344235925245675, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.002, "humaneval_pass1": 0.03048780487804878, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02122820318423048, "gsm8k_exact_match_flexible": 0.025018953752843062, "flores_mean_bpb_common": 1.3344235925245678, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-punct-balanced-bpe": { "run": "small-128k-punct-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26646655781, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8583778693432756, "flores_mean_bpb": 1.3317774057090888, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3317774057090888, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-punct-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-punct-balanced-bpe_blimp_code_bpb.json; no small-128k-punct-balanced-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-punct-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-punct-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-punct-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-punct-english-bpe": { "run": "small-128k-punct-english-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24756548948, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8779310065917464, "flores_mean_bpb": 1.4025077988234407, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.402507798823441, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-punct-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-punct-english-bpe_blimp_code_bpb.json; no small-128k-punct-english-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-punct-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-punct-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-punct-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-bpe": { "run": "small-128k-rightalign-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830565735, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8476906713771896, "flores_mean_bpb": 1.3241330136201725, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3241330136201728, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-bpe_blimp_code_bpb.json; no small-128k-rightalign-balanced-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-rightalign-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-nfc-bpe": { "run": "small-128k-rightalign-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25831056274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8475073186072606, "flores_mean_bpb": 1.3218437584330118, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.3218437584330123, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-rightalign-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no small-128k-rightalign-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-unigram": { "run": "small-128k-rightalign-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725040077, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8831570596770636, "flores_mean_bpb": 1.361640141186164, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.361640141186164, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-unigram_blimp_code_bpb.json; no small-128k-rightalign-balanced-unigram_code_bpb.json", "mbpp_pass1": "no small-128k-rightalign-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-superbpe-gpt4o-balanced": { "run": "small-128k-superbpe-gpt4o-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 380584418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26637388932, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8516819488448236, "blimp_acc": 0.5957014925373133, "code_bpb_mean": 0.6307925376771104, "blimp_scoring": null, "flores_mean_bpb": 1.3385163447738013, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.006, "humaneval_pass1": 0.012195121951219513, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.009097801364670205, "gsm8k_exact_match_flexible": 0.013646702047005308, "flores_mean_bpb_common": 1.3385163447738013, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/small-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-apertus": { "run": "tiny-128k-apertus", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 226754794 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 23311828168, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.9355125749278125, "flores_mean_bpb": 1.4191660344874155, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4191660344874155, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-apertus_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-apertus_blimp_code_bpb.json; no tiny-128k-apertus_code_bpb.json", "mbpp_pass1": "no tiny-128k-apertus_code_v2.json", "humaneval_pass1": "no tiny-128k-apertus_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-apertus_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-bpe": { "run": "tiny-128k-claude-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471449933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9203617131329531, "flores_mean_bpb": 1.4138344975543817, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4138344975543815, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-bpe_blimp_code_bpb.json; no tiny-128k-claude-balanced-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-nfc-bpe": { "run": "tiny-128k-claude-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471396541, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.92068400302786, "flores_mean_bpb": 1.41133643188519, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.41133643188519, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-claude-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-unigram": { "run": "tiny-128k-claude-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22708535275, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9638156719834017, "flores_mean_bpb": 1.4607471746132827, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.460747174613283, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-unigram_blimp_code_bpb.json; no tiny-128k-claude-balanced-unigram_code_bpb.json", "mbpp_pass1": "no tiny-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-english-bpe": { "run": "tiny-128k-claude-english-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24979881010, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9462504048449769, "flores_mean_bpb": 1.5165879981054353, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.516587998105435, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-english-bpe_blimp_code_bpb.json; no tiny-128k-claude-english-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-bpe": { "run": "tiny-128k-gpt4o-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830345493, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9171902254705996, "flores_mean_bpb": 1.4143278220394313, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4143278220394315, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-nfc-bpe": { "run": "tiny-128k-gpt4o-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830839369, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9169944413597259, "flores_mean_bpb": 1.4118278134972433, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4118278134972433, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-unigram": { "run": "tiny-128k-gpt4o-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725477224, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9614550260734541, "flores_mean_bpb": 1.4604569743152251, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4604569743152251, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-unigram_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-unigram_code_bpb.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-code-bpe": { "run": "tiny-128k-gpt4o-code-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22625023513, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9463340638979787, "flores_mean_bpb": 1.5060894719396118, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.5060894719396116, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-code-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-code-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-code-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-gpt4o-code-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-code-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-code-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-english-bpe": { "run": "tiny-128k-gpt4o-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24697073597, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9515298429772641 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-english-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-english-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-english-bpe.json and no full-devtest result", "mbpp_pass1": "no tiny-128k-gpt4o-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-llama3": { "run": "tiny-128k-llama3", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22587181933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.9359259465717317, "flores_mean_bpb": 1.4524403663725347, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4524403663725345, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-llama3_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-llama3_blimp_code_bpb.json; no tiny-128k-llama3_code_bpb.json", "mbpp_pass1": "no tiny-128k-llama3_code_v2.json", "humaneval_pass1": "no tiny-128k-llama3_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-llama3_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-pabpe-nfc-gpt4-balanced": { "run": "tiny-128k-pabpe-nfc-gpt4-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 221839594 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.9584538663771468, "blimp_acc": 0.5613283582089551, "code_bpb_mean": 0.7416731447732372, "blimp_scoring": null, "flores_mean_bpb": 1.424384551843815, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.0, "humaneval_pass1": 0.0, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.01288855193328279, "gsm8k_exact_match_flexible": 0.02577710386656558, "flores_mean_bpb_common": 1.424384551843815, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-punct-balanced-bpe": { "run": "tiny-128k-punct-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26646655781, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9268041882882746, "flores_mean_bpb": 1.4236233050849518, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4236233050849518, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-punct-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-punct-balanced-bpe_blimp_code_bpb.json; no tiny-128k-punct-balanced-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-punct-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-punct-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-punct-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-punct-english-bpe": { "run": "tiny-128k-punct-english-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24756548948, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9492255185989006, "flores_mean_bpb": 1.5162535370033292, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.5162535370033292, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-punct-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-punct-english-bpe_blimp_code_bpb.json; no tiny-128k-punct-english-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-punct-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-punct-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-punct-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-bpe": { "run": "tiny-128k-rightalign-balanced-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830565735, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9179302641444103, "flores_mean_bpb": 1.416250618657187, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.4162506186571866, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-bpe_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-nfc-bpe": { "run": "tiny-128k-rightalign-balanced-nfc-bpe", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25831056274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9177500705611673, "flores_mean_bpb": 1.413507916830717, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.413507916830717, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-nfc-bpe_code_bpb.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-unigram": { "run": "tiny-128k-rightalign-balanced-unigram", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725040077, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9614013673266325, "flores_mean_bpb": 1.460022751190828, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "flores_mean_bpb_common": 1.460022751190828, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-unigram_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-unigram_code_bpb.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-superbpe-gpt4o-balanced": { "run": "tiny-128k-superbpe-gpt4o-balanced", "works": [ "paper_a_cross_scale", "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 222134506 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26637388932, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.9208942139791034, "blimp_acc": 0.6045223880597016, "code_bpb_mean": 0.6974649709299585, "blimp_scoring": null, "flores_mean_bpb": 1.435066610690789, "flores_n_languages": 31, "flores_basis": "full1012", "flores_sentences_per_language": 1012, "flores_language_scope": "trained31", "mbpp_pass1": 0.004, "humaneval_pass1": 0.018292682926829267, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.008339651250947688, "gsm8k_exact_match_flexible": 0.022744503411675512, "flores_mean_bpb_common": 1.4350666106907888, "flores_n_languages_common": 31, "flores_common_basis": "full1012/trained31" }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a0229/cmeister/tokenizer-lm/checkpoints/tiny-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } } } }