Hugging Face's logo

Papatpong
/
Walker

Text Generation
English
Model card Files
xet
 Community
Papatpong commited on
Commit
f8e244b
·
verified ·
1 Parent(s): c5447df

Upload checkpoints/branch47750_ckpt_ongoing.pt.metadata.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. checkpoints/branch47750_ckpt_ongoing.pt.metadata.json +1 -1
checkpoints/branch47750_ckpt_ongoing.pt.metadata.json CHANGED
@@ -1 +1 @@
1
- {"version": 1, "step": 55500, "run_state": {"version": 2, "source_sha256": "d5df4bd22a1973ffa77e86bb08ce3b4b892b58ef1f8a1ad9593b06e54ec14bcc", "config": {"seed": 0, "tokenizer": "gpt2", "vocab": 50257, "D": 2048, "L": 16, "n_head": 16, "mult": 2.5, "steps": 60000, "B": 1, "t0": 8192, "t1": 8192, "eval_t": 4096, "lr": 0.0003, "lr_floor": 2e-05, "warmup": 500, "lr_schedule": "tokens", "schedule_steps": 122071, "lr_total_tokens": 15000000000, "lr_warmup_tokens": 1, "train_phase": 1, "resume_schedule": "override", "adapter_mult": 1.0, "phase_mult": 1.0, "r_adapter_mult": 1.0, "accum": 15, "amp": true, "ckpt": false, "walk_norm": 0.3, "walk_norm_mode": "clip", "walk_ablate_eval": false, "traj_smooth": 0.0, "traj_acc": 0.0, "doremi": false, "doremi_eta": 0.05, "doremi_warmup": 0.05, "fp4": true, "v41_engram": false, "v41_engram_buckets": 32768, "v41_engram_rank": 64, "v41_ced": false, "v41_csa2": false, "v41_csa2_alpha_start": 0.0, "v41_csa2_alpha_end": 1.0, "v41_csa2_ramp_steps": 0, "quant_backend": "ste", "no_walk": false, "ni": false, "ni_mode": "final", "ni_steps": 1, "ni_trajectory": true, "ni_slots": 4, "ni_rank": 32, "ni_width_mult": 1, "ni_bridge": false, "ni_bridge_weight": 0.01, "ni_chunk_size": 128, "ni_layer": -1, "ni_layers": "", "lmm": false, "lmm_rank": 128, "lmm_layers": "", "lmm_write": "delta", "lmm_lesson_source": "lookup", "lmm_eta": 0.25, "lmm_decay": 0.01, "lmm_gate_bias": -4.0, "lexical_semantic": false, "lexical_semantic_rank": 256, "alibi": false, "rope": true, "hf_ckpt": "Papatpong/Walker", "max_t": 16384, "optimizer": "adamw", "local_conv": false, "local_conv_rank": 128, "local_conv_kernels": "3,7,15", "local_conv_every": 1, "local_conv_depth": 3, "local_conv_stage2_kernels": "7,15,31", "local_conv_stage3_kernels": "15,31,63", "local_conv_gate_init": 0.0, "local_conv_gate_bootstrap": 0.0, "local_conv_reseed_zero_gates": false, "hybrid": true, "freeze_walk_at": 0, "eval_t2": 8192, "log_every": 250, "trend_every": 50, "ckpt_every": 250, "grad_clip": 2.0, "mem_mult": 20.0, "n_eval_batches": 20, "ema_eval": true, "gmem": false, "gmem_weight": 0.1, "gmem_every": 1, "gmem_batch": 16, "gmem_writes": 2, "gmem_r": 4, "gmem_lr": 0.001, "gmem_walker_grad": false, "gmem_lora_rank": 0, "gmem_trace_every": 100, "ema_decay": 0.999, "data_dir": "", "stream": "", "stream_preset": "", "stream_mix": "[{\"budget_tokens\":750000000,\"dataset\":\"HuggingFaceTB/smollm-corpus:cosmopedia-v2\",\"domain_key\":\"prompt\",\"domains\":3,\"name\":\"cosmopedia_v2\",\"revision\":\"3ba9d60\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":4500000000,\"dataset\":\"wikimedia/wikipedia:20231101.en\",\"domain_key\":\"title\",\"domains\":4,\"name\":\"wikipedia_en\",\"revision\":\"b04c8d1ceb2f5cd4588862100d08de323dccfbaa\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":2550000000,\"dataset\":\"OctoThinker/MegaMath-Web-Pro-Max\",\"domain_key\":\"id\",\"domains\":4,\"name\":\"megamath_web_pro_max\",\"revision\":\"0194035ca21e11ce60c4693fd2607e621dec9324\",\"text_field\":\"text\",\"weight\":0.1},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-RQA\",\"name\":\"stem_rqa\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1.2\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Fact-Seeking\",\"name\":\"fact_seeking\",\"revision\":\"807afc1fa65c441d46ebc7d9b95295a35499a527\",\"text_field\":\"text\",\"weight\":0.05},{\"budget_tokens\":600000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-InfiniByte-Reasoning\",\"name\":\"infinibyte\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Math-Textbooks\",\"name\":\"math_textbooks\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:vendor\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_vendor\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Math-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data\",\"message_field\":\"messages\",\"name\":\"sft_math\",\"revision\":\"8e793210e175b6406c752a870f585f62de98c0d3\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.09},{\"budget_tokens\":380000000,\"dataset\":\"HuggingFaceTB/smoltalk\",\"dedup\":\"first_user\",\"domain_key\":\"category\",\"domains\":3,\"file_selector\":\"smol-magpie-ultra/train\",\"filters\":{\"category\":[\"not_in\",[\"coding\",\"math\"]],\"quality\":[\"not_in\",[\"poor\",\"very poor\"]]},\"message_field\":\"messages\",\"name\":\"sft_chat\",\"revision\":\"5feaf2fd3ffca7c237fc38d1861bc30365d48ffa\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":2100000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-Wiki-Rewrite\",\"name\":\"wiki_rewrite\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.14},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Scientific-Coding\",\"name\":\"scientific_coding\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-SFT-Instruction-Following-Chat-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data/reasoning_off\",\"message_field\":\"messages\",\"name\":\"sft_if_chat\",\"revision\":\"1a9454ed054b8544503ab8d8c0a519d141a44c5b\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":700000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:rqa\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_rqa\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":160000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:so\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_so\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1500000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"dedup\":\"text_head\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-STEM-SFT\",\"name\":\"stem_sft\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.1}]", "stream_mix_total_tokens": 15000000000, "stream_val_source": 0, "stream_text_field": "text", "stream_revision": "", "stream_nd": 32, "stream_domain": "url", "stream_buf": 1000000, "code_stream": "", "code_stream_revision": "", "code_data_dir": "", "code_stream_nd": 128, "code_stream_buf": 500000, "code_text_field": "content", "code_repo_field": "repo_name", "code_path_field": "path", "code_lang_field": "language", "code_group_repo": true, "code_lang_filter": "", "code_share": 0.0, "code_share_start": 0.0, "code_ramp_tokens": 100000000, "code_val_tokens": 0, "code_val_path": "code_val_phase_b.bin", "code_best_metric": "mix", "cot_url": "", "cot_path": "", "cot_tokens": 2000000, "cot_share_max": 0.0, "fw_val_tokens": 0, "fw_val_path": "fw_val.bin", "mix_val_tokens_per_source": 524288, "mix_val_path": "/marimo/walker_runs/walker-sft2-continue-03/validation/mix_a2fb6980d8737978_524288", "ckpt_path": "/marimo/walker_runs/walker-sft2-continue-03/ckpt_ongoing.pt", "init_ckpt": "", "init_allow_partial": false, "phase_ckpt": "", "phase_structural": false, "phase_ce_tolerance_pct": 2.0, "phase_reference_ce": NaN, "phase_allow_unchecked": false, "phase_gate_batches": 4, "arch_phase": "phase_b", "phase_ablate_eval": false, "v41_ablate_eval": false, "phase_ablate_batches": 2, "semantic_wd": true, "draft": false, "profile": true, "stream_mix_signature": "632bd873d5e4296a", "schedule_origin_step": 0}, "schedule": {"version": 1, "kind": "tokens", "total_steps": 122071, "stop_steps": 60000, "peak_lr": 0.0003, "floor": 2e-05, "warmup_steps": 500, "total_tokens": 15000000000, "warmup_tokens": 1, "origin_step": 0}, "train_phase": 1, "architecture": {"version": 1, "phase": "phase_b", "shape": {"vocab": 50257, "D": 2048, "L": 16, "heads": 16, "mult": 2.5, "max_t": 16384, "rope": true, "alibi": false, "hybrid": true, "local_conv": false, "local_conv_rank": 128, "local_conv_kernels": "3,7,15", "local_conv_every": 1, "local_conv_depth": 3, "local_conv_stage2_kernels": "7,15,31", "local_conv_stage3_kernels": "15,31,63", "lmm": false, "lmm_rank": 128, "lmm_layers": "", "lmm_write": "delta", "lmm_lesson_source": "lookup", "lexical_semantic": false, "lexical_semantic_rank": 256, "v41_engram": false, "v41_engram_buckets": 32768, "v41_engram_rank": 64, "v41_ced": false, "v41_csa2": false, "v41_csa2_mode": "blend", "walk_cycle_guard": false, "walk_cycle_escape": false, "walk_cycle_rank": 32, "walk_cycle_lags": "1,2,4,8", "pause_tokens": 0, "pause_every": 0, "pause_huginn": false, "pause_rank": 0, "r_adapter": false, "r_start": 0, "r_end": 0, "r_max": 0, "r_backprop": 0}, "env": {"TB_ATTN_EVERY": "1", "TB_HYBRID": "1", "TB_NO_WALK": "0", "TB_MLP_COMPACT": "0", "TB_MLP_COMPACT_EQUAL": "0", "TB_WALK_SHARED": "0", "TB_WALK_LOOP": "1", "TB_WALK_NORM": "0.3", "TB_WALK_NORM_MODE": "clip", "TB_WALK_HEADS": "4", "TB_WALK_FP32": "1", "TB_WALK_MIX": "1", "TB_WALK_MIX_DEPTH": "1", "TB_WALK_MIX_GAIN": "1.0", "TB_WALK_GLOB": "1", "TB_WALK_COUPLE": "1", "TB_WALK_GATE": "1", "TB_WALK_SELECTIVE_MEM": "", "TB_WALK_GEOM": "", "TB_WALK_PLACE": "", "TB_WALK_HURST": "", "TB_WALK_AFFINE": "", "TB_WALK_MOTIF": "", "TB_WALK_MOTIF_COUNT": "", "TB_WALK_MOTIF_RANK": "", "TB_WALK_MOTIF_PRED": "", "TB_WALK_MOTIF_V2": "", "TB_WALK_MOTIF_ORTHO": "", "TB_WALK_MOTIF_SHARED": "", "TB_WALK_MOTIF_CONF": "", "TB_WALK_MOTIF_TOPK": "", "TB_WALK_MOTIF_MOD": "", "TB_WDR": "", "TB_WDR_LAYER": "", "TB_WDR_RANK": "", "TB_WALK_CYCLE_GUARD": "0", "TB_WALK_CYCLE_ESCAPE": "0", "TB_WALK_CYCLE_RANK": "", "TB_WALK_CYCLE_LAGS": "", "TB_WALK_CYCLE_THRESHOLD": "", "TB_WALK_CYCLE_PERSISTENCE": "", "TB_WALK_CYCLE_MAX_STRENGTH": "", "TB_WALK_CYCLE_OPEN_TARGET": "", "TB_WALK_CYCLE_OPEN_WEIGHT": "", "TB_WALK_SUBSPACE_RANK": "0", "TB_WALK_HIER_EVIDENCE": "0", "TB_WALK_EVIDENCE": "0", "TB_WALK_DATA_TRAJ": "0", "TB_WALK_HIER_K": "", "TB_WALK_HIER_TRAJ": "", "TB_WALK_HIER_CHANNEL_GATE": "", "TB_WALK_HIER_LEARNED": "", "TB_WALK_HIER_AFFINE_FREE": "", "TB_WALK_HIER_DEPTH_RMS": "", "TB_WALK_HIER_ARCHIVE": "", "TB_WALK_HIER_ARCHIVE_CHUNK": "", "TB_WALK_HIER_ARCHIVE_TOPK": "", "TB_WALK_PRED": "0", "TB_WALK_PRED_CE": "0", "TB_WALK_SUBSPACE_GATED": "0", "TB_WALK_SUBSPACE_CONTENT": "0", "TB_WALK_MEMORY_CHUNK": "0", "TB_WALK_PATH_MIXER": "0", "TB_WALK_DUAL_PATH_MIXER": "0", "TB_TIE_EMB_HEAD": "0", "TB_POS_EMB": "1", "TB_MEM": "0", "TB_NI": "0", "TB_NI_MODE": "", "TB_NI_STEPS": "", "TB_NI_RANK": "", "TB_NI_WIDTH_MULT": "", "TB_NI_LAYERS": "", "TB_PHASE_B_SUPERSET": "1", "TB_PHASE_WALK_LAYERS": "all", "TB_PHASE_WALK_RANK": "32", "TB_PHASE_WALK_K": "2", "TB_PHASE_WALK_DATA_TRAJ": "1", "TB_PHASE_WALK_TRAJ": "full", "TB_PHASE_HIER_SCALES": "8,64", "TB_PHASE_WALK_NORM": "0.05", "TB_PHASE_WALK_NORM_MODE": "clip", "TB_PHASE_ARCHIVE_CHUNK": "64", "TB_PHASE_ARCHIVE_TOPK": "1", "TB_TRAJ_REG": "0", "TB_TRAJ_REG_RANK": "", "TB_TRAJ_REG_HORIZON": "", "TB_KMERGE": "0", "TB_KMERGE_K": "", "TB_KMERGE_PREFIX_FRAC": "", "TB_LEXICAL_SEMANTIC": "0", "TB_LEXICAL_SEMANTIC_RANK": "256", "TB_SEMANTIC_WD": "1", "TB_V41_ENGRAM": "0", "TB_V41_ENGRAM_BUCKETS": "", "TB_V41_ENGRAM_RANK": "", "TB_V41_KV_FP4": "0", "TB_V41_CED": "0", "TB_V41_CSA2": "0", "TB_PAUSE_N": "", "TB_PAUSE_EVERY": "", "TB_PAUSE_HUGINN": "", "TB_PAUSE_RANK": "", "TB_R_ENABLE": "0", "TB_R_START": "", "TB_R_END": "", "TB_R_MEAN": "", "TB_R_MAX": "", "TB_R_BACKPROP": "", "TB_R_INSTR_ONLY": "", "TB_R_KV_SHARE": "", "TB_R_UPDATE": "", "TB_R_GAIN": "", "TB_WALK_ADAPT_SCALE": "", "TB_WALK_ADAPT_LAGS": "", "TB_WALK_ADAPT_RANK": "", "TB_WALK_CONV_BASELINE": "", "TB_LOOP_ENABLE": "0", "TB_LOOP_START": "", "TB_LOOP_END": "", "TB_LOOP_PASSES": ""}}, "data": {"version": 4, "env": {"TB_STREAM": "", "TB_STREAM_REVISION": "", "TB_STREAM_ND": "", "TB_STREAM_DOMAIN": "", "TB_STREAM_TEXT_FIELD": "", "TB_STREAM_PRESET": "", "TB_STREAM_MIX": "[{\"budget_tokens\":750000000,\"dataset\":\"HuggingFaceTB/smollm-corpus:cosmopedia-v2\",\"domain_key\":\"prompt\",\"domains\":3,\"name\":\"cosmopedia_v2\",\"revision\":\"3ba9d60\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":4500000000,\"dataset\":\"wikimedia/wikipedia:20231101.en\",\"domain_key\":\"title\",\"domains\":4,\"name\":\"wikipedia_en\",\"revision\":\"b04c8d1ceb2f5cd4588862100d08de323dccfbaa\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":2550000000,\"dataset\":\"OctoThinker/MegaMath-Web-Pro-Max\",\"domain_key\":\"id\",\"domains\":4,\"name\":\"megamath_web_pro_max\",\"revision\":\"0194035ca21e11ce60c4693fd2607e621dec9324\",\"text_field\":\"text\",\"weight\":0.1},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-RQA\",\"name\":\"stem_rqa\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1.2\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Fact-Seeking\",\"name\":\"fact_seeking\",\"revision\":\"807afc1fa65c441d46ebc7d9b95295a35499a527\",\"text_field\":\"text\",\"weight\":0.05},{\"budget_tokens\":600000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-InfiniByte-Reasoning\",\"name\":\"infinibyte\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Math-Textbooks\",\"name\":\"math_textbooks\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:vendor\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_vendor\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Math-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data\",\"message_field\":\"messages\",\"name\":\"sft_math\",\"revision\":\"8e793210e175b6406c752a870f585f62de98c0d3\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.09},{\"budget_tokens\":380000000,\"dataset\":\"HuggingFaceTB/smoltalk\",\"dedup\":\"first_user\",\"domain_key\":\"category\",\"domains\":3,\"file_selector\":\"smol-magpie-ultra/train\",\"filters\":{\"category\":[\"not_in\",[\"coding\",\"math\"]],\"quality\":[\"not_in\",[\"poor\",\"very poor\"]]},\"message_field\":\"messages\",\"name\":\"sft_chat\",\"revision\":\"5feaf2fd3ffca7c237fc38d1861bc30365d48ffa\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":2100000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-Wiki-Rewrite\",\"name\":\"wiki_rewrite\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.14},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Scientific-Coding\",\"name\":\"scientific_coding\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-SFT-Instruction-Following-Chat-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data/reasoning_off\",\"message_field\":\"messages\",\"name\":\"sft_if_chat\",\"revision\":\"1a9454ed054b8544503ab8d8c0a519d141a44c5b\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":700000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:rqa\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_rqa\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":160000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:so\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_so\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1500000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"dedup\":\"text_head\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-STEM-SFT\",\"name\":\"stem_sft\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.1}]", "TB_STREAM_VAL_SOURCE": "", "TB_STREAM_SHUFFLE_BUFFER": "0", "TB_STREAM_SHUFFLE_SEED": "0", "TB_CODE_STREAM": "", "TB_CODE_STREAM_REVISION": "", "TB_CODE_DATA_DIR": "", "TB_CODE_STREAM_ND": "", "TB_CODE_TEXT_FIELD": "", "TB_CODE_REPO_FIELD": "", "TB_CODE_PATH_FIELD": "", "TB_CODE_LANG_FIELD": "", "TB_CODE_GROUP_REPO": "", "TB_CODE_LANG_FILTER": "", "TB_CODE_SHARE": "", "TB_CODE_SHARE_START": "", "TB_CODE_RAMP_TOKENS": "", "TB_CODE_VAL_TOKENS": "", "TB_CODE_VAL_PATH": "", "TB_COT_URL": "", "TB_COT_PATH": "", "TB_SFT_MASK": "1"}, "stream_mix_signature": "632bd873d5e4296a", "tokenizer": "gpt2", "vocab": 50257, "packing": {"fineweb": "eos-v1", "code": "eos-v1", "windowing": "contiguous-no-drop-v2"}}, "runtime": {"TB_COMPILE": "0", "TB_COMPILE_DYNAMIC": "0", "TB_ACTIVATION_CKPT_STRIDE": "1", "TB_ACTIVATION_CKPT_FULL_AT_T": "0", "TB_WALK_LOOP_CKPT": "0", "TB_ATTN_WINDOW_CHUNK": "512", "TB_PHASE_ASSOC_CHUNK": "0", "TB_WALK_ASSOC_CHUNK": "0", "TB_LOCAL_CONV_FUSE_KERNELS": "1", "TB_WALK_MIX_TENSORCORE": "0", "TB_METRIC_EVERY": "1", "TB_LOG_EVERY": "250", "TB_TREND_EVERY": "50", "TB_CKPT_EVERY": "250", "TB_BASELINE_VAL_ON_RESUME": "1", "TB_WALK_ABLATE_EVAL": "0", "TB_PHASE_ABLATE_EVAL": "0", "TB_V41_ABLATE_EVAL": "0", "TB_GRAD_CLIP": "2.0", "TB_ADAPTER_MULT": "1.0", "TB_PHASE_MULT": "1.0", "TB_LOCAL_CONV_MULT": "1.0", "TB_LEXICAL_SEMANTIC_MULT": "1.0", "TB_LEXICAL_SEMANTIC_AUX_WEIGHT": "0.05", "TB_LEXICAL_SEMANTIC_STRIDE": "32", "TB_LEXICAL_SEMANTIC_SCALE": "1.0", "TB_LEXICAL_SEMANTIC_TEMPERATURE": "0.1", "TB_WALK_FORWARD_SCALE": "1.0", "TB_PHASE_WALK_FORWARD_SCALE": "0.0", "TB_WALK_INPUT_GRAD_SCALE": "1.0", "TB_PHASE_WALK_INPUT_GRAD_SCALE": "1.0", "TB_PHASE_PRED_WEIGHT": "0.0", "TB_PHASE_PRED_SCALE": "0.0", "TB_R_RESID_LOG": "", "TB_R_HALT_EPS": "", "TB_R_HALT_MIN": "", "TB_R_ADAPTER_MULT": "", "TB_EMA_EVAL": "1", "TB_EMA_DECAY": "0.999", "TB_OPT_KAHAN": "1", "TB_R_SAMPLE": "", "TB_LR_COOLDOWN_START": "47750", "TB_LR_COOLDOWN_FLOOR": "2e-05", "TB_BOTTOM_ISOLATE": "1", "TB_BOTTOM_ISOLATE_RAMP": "500", "TB_BOTTOM_LR_MULT": "0.25", "TB_HF_CKPT_BRANCH": "47750"}}}
 
1
+ {"version": 1, "step": 55750, "run_state": {"version": 2, "source_sha256": "d5df4bd22a1973ffa77e86bb08ce3b4b892b58ef1f8a1ad9593b06e54ec14bcc", "config": {"seed": 0, "tokenizer": "gpt2", "vocab": 50257, "D": 2048, "L": 16, "n_head": 16, "mult": 2.5, "steps": 60000, "B": 1, "t0": 8192, "t1": 8192, "eval_t": 4096, "lr": 0.0003, "lr_floor": 2e-05, "warmup": 500, "lr_schedule": "tokens", "schedule_steps": 122071, "lr_total_tokens": 15000000000, "lr_warmup_tokens": 1, "train_phase": 1, "resume_schedule": "override", "adapter_mult": 1.0, "phase_mult": 1.0, "r_adapter_mult": 1.0, "accum": 15, "amp": true, "ckpt": false, "walk_norm": 0.3, "walk_norm_mode": "clip", "walk_ablate_eval": false, "traj_smooth": 0.0, "traj_acc": 0.0, "doremi": false, "doremi_eta": 0.05, "doremi_warmup": 0.05, "fp4": true, "v41_engram": false, "v41_engram_buckets": 32768, "v41_engram_rank": 64, "v41_ced": false, "v41_csa2": false, "v41_csa2_alpha_start": 0.0, "v41_csa2_alpha_end": 1.0, "v41_csa2_ramp_steps": 0, "quant_backend": "ste", "no_walk": false, "ni": false, "ni_mode": "final", "ni_steps": 1, "ni_trajectory": true, "ni_slots": 4, "ni_rank": 32, "ni_width_mult": 1, "ni_bridge": false, "ni_bridge_weight": 0.01, "ni_chunk_size": 128, "ni_layer": -1, "ni_layers": "", "lmm": false, "lmm_rank": 128, "lmm_layers": "", "lmm_write": "delta", "lmm_lesson_source": "lookup", "lmm_eta": 0.25, "lmm_decay": 0.01, "lmm_gate_bias": -4.0, "lexical_semantic": false, "lexical_semantic_rank": 256, "alibi": false, "rope": true, "hf_ckpt": "Papatpong/Walker", "max_t": 16384, "optimizer": "adamw", "local_conv": false, "local_conv_rank": 128, "local_conv_kernels": "3,7,15", "local_conv_every": 1, "local_conv_depth": 3, "local_conv_stage2_kernels": "7,15,31", "local_conv_stage3_kernels": "15,31,63", "local_conv_gate_init": 0.0, "local_conv_gate_bootstrap": 0.0, "local_conv_reseed_zero_gates": false, "hybrid": true, "freeze_walk_at": 0, "eval_t2": 8192, "log_every": 250, "trend_every": 50, "ckpt_every": 250, "grad_clip": 2.0, "mem_mult": 20.0, "n_eval_batches": 20, "ema_eval": true, "gmem": false, "gmem_weight": 0.1, "gmem_every": 1, "gmem_batch": 16, "gmem_writes": 2, "gmem_r": 4, "gmem_lr": 0.001, "gmem_walker_grad": false, "gmem_lora_rank": 0, "gmem_trace_every": 100, "ema_decay": 0.999, "data_dir": "", "stream": "", "stream_preset": "", "stream_mix": "[{\"budget_tokens\":750000000,\"dataset\":\"HuggingFaceTB/smollm-corpus:cosmopedia-v2\",\"domain_key\":\"prompt\",\"domains\":3,\"name\":\"cosmopedia_v2\",\"revision\":\"3ba9d60\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":4500000000,\"dataset\":\"wikimedia/wikipedia:20231101.en\",\"domain_key\":\"title\",\"domains\":4,\"name\":\"wikipedia_en\",\"revision\":\"b04c8d1ceb2f5cd4588862100d08de323dccfbaa\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":2550000000,\"dataset\":\"OctoThinker/MegaMath-Web-Pro-Max\",\"domain_key\":\"id\",\"domains\":4,\"name\":\"megamath_web_pro_max\",\"revision\":\"0194035ca21e11ce60c4693fd2607e621dec9324\",\"text_field\":\"text\",\"weight\":0.1},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-RQA\",\"name\":\"stem_rqa\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1.2\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Fact-Seeking\",\"name\":\"fact_seeking\",\"revision\":\"807afc1fa65c441d46ebc7d9b95295a35499a527\",\"text_field\":\"text\",\"weight\":0.05},{\"budget_tokens\":600000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-InfiniByte-Reasoning\",\"name\":\"infinibyte\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Math-Textbooks\",\"name\":\"math_textbooks\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:vendor\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_vendor\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Math-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data\",\"message_field\":\"messages\",\"name\":\"sft_math\",\"revision\":\"8e793210e175b6406c752a870f585f62de98c0d3\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.09},{\"budget_tokens\":380000000,\"dataset\":\"HuggingFaceTB/smoltalk\",\"dedup\":\"first_user\",\"domain_key\":\"category\",\"domains\":3,\"file_selector\":\"smol-magpie-ultra/train\",\"filters\":{\"category\":[\"not_in\",[\"coding\",\"math\"]],\"quality\":[\"not_in\",[\"poor\",\"very poor\"]]},\"message_field\":\"messages\",\"name\":\"sft_chat\",\"revision\":\"5feaf2fd3ffca7c237fc38d1861bc30365d48ffa\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":2100000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-Wiki-Rewrite\",\"name\":\"wiki_rewrite\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.14},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Scientific-Coding\",\"name\":\"scientific_coding\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-SFT-Instruction-Following-Chat-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data/reasoning_off\",\"message_field\":\"messages\",\"name\":\"sft_if_chat\",\"revision\":\"1a9454ed054b8544503ab8d8c0a519d141a44c5b\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":700000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:rqa\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_rqa\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":160000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:so\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_so\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1500000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"dedup\":\"text_head\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-STEM-SFT\",\"name\":\"stem_sft\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.1}]", "stream_mix_total_tokens": 15000000000, "stream_val_source": 0, "stream_text_field": "text", "stream_revision": "", "stream_nd": 32, "stream_domain": "url", "stream_buf": 1000000, "code_stream": "", "code_stream_revision": "", "code_data_dir": "", "code_stream_nd": 128, "code_stream_buf": 500000, "code_text_field": "content", "code_repo_field": "repo_name", "code_path_field": "path", "code_lang_field": "language", "code_group_repo": true, "code_lang_filter": "", "code_share": 0.0, "code_share_start": 0.0, "code_ramp_tokens": 100000000, "code_val_tokens": 0, "code_val_path": "code_val_phase_b.bin", "code_best_metric": "mix", "cot_url": "", "cot_path": "", "cot_tokens": 2000000, "cot_share_max": 0.0, "fw_val_tokens": 0, "fw_val_path": "fw_val.bin", "mix_val_tokens_per_source": 524288, "mix_val_path": "/marimo/walker_runs/walker-sft2-continue-03/validation/mix_a2fb6980d8737978_524288", "ckpt_path": "/marimo/walker_runs/walker-sft2-continue-03/ckpt_ongoing.pt", "init_ckpt": "", "init_allow_partial": false, "phase_ckpt": "", "phase_structural": false, "phase_ce_tolerance_pct": 2.0, "phase_reference_ce": NaN, "phase_allow_unchecked": false, "phase_gate_batches": 4, "arch_phase": "phase_b", "phase_ablate_eval": false, "v41_ablate_eval": false, "phase_ablate_batches": 2, "semantic_wd": true, "draft": false, "profile": true, "stream_mix_signature": "632bd873d5e4296a", "schedule_origin_step": 0}, "schedule": {"version": 1, "kind": "tokens", "total_steps": 122071, "stop_steps": 60000, "peak_lr": 0.0003, "floor": 2e-05, "warmup_steps": 500, "total_tokens": 15000000000, "warmup_tokens": 1, "origin_step": 0}, "train_phase": 1, "architecture": {"version": 1, "phase": "phase_b", "shape": {"vocab": 50257, "D": 2048, "L": 16, "heads": 16, "mult": 2.5, "max_t": 16384, "rope": true, "alibi": false, "hybrid": true, "local_conv": false, "local_conv_rank": 128, "local_conv_kernels": "3,7,15", "local_conv_every": 1, "local_conv_depth": 3, "local_conv_stage2_kernels": "7,15,31", "local_conv_stage3_kernels": "15,31,63", "lmm": false, "lmm_rank": 128, "lmm_layers": "", "lmm_write": "delta", "lmm_lesson_source": "lookup", "lexical_semantic": false, "lexical_semantic_rank": 256, "v41_engram": false, "v41_engram_buckets": 32768, "v41_engram_rank": 64, "v41_ced": false, "v41_csa2": false, "v41_csa2_mode": "blend", "walk_cycle_guard": false, "walk_cycle_escape": false, "walk_cycle_rank": 32, "walk_cycle_lags": "1,2,4,8", "pause_tokens": 0, "pause_every": 0, "pause_huginn": false, "pause_rank": 0, "r_adapter": false, "r_start": 0, "r_end": 0, "r_max": 0, "r_backprop": 0}, "env": {"TB_ATTN_EVERY": "1", "TB_HYBRID": "1", "TB_NO_WALK": "0", "TB_MLP_COMPACT": "0", "TB_MLP_COMPACT_EQUAL": "0", "TB_WALK_SHARED": "0", "TB_WALK_LOOP": "1", "TB_WALK_NORM": "0.3", "TB_WALK_NORM_MODE": "clip", "TB_WALK_HEADS": "4", "TB_WALK_FP32": "1", "TB_WALK_MIX": "1", "TB_WALK_MIX_DEPTH": "1", "TB_WALK_MIX_GAIN": "1.0", "TB_WALK_GLOB": "1", "TB_WALK_COUPLE": "1", "TB_WALK_GATE": "1", "TB_WALK_SELECTIVE_MEM": "", "TB_WALK_GEOM": "", "TB_WALK_PLACE": "", "TB_WALK_HURST": "", "TB_WALK_AFFINE": "", "TB_WALK_MOTIF": "", "TB_WALK_MOTIF_COUNT": "", "TB_WALK_MOTIF_RANK": "", "TB_WALK_MOTIF_PRED": "", "TB_WALK_MOTIF_V2": "", "TB_WALK_MOTIF_ORTHO": "", "TB_WALK_MOTIF_SHARED": "", "TB_WALK_MOTIF_CONF": "", "TB_WALK_MOTIF_TOPK": "", "TB_WALK_MOTIF_MOD": "", "TB_WDR": "", "TB_WDR_LAYER": "", "TB_WDR_RANK": "", "TB_WALK_CYCLE_GUARD": "0", "TB_WALK_CYCLE_ESCAPE": "0", "TB_WALK_CYCLE_RANK": "", "TB_WALK_CYCLE_LAGS": "", "TB_WALK_CYCLE_THRESHOLD": "", "TB_WALK_CYCLE_PERSISTENCE": "", "TB_WALK_CYCLE_MAX_STRENGTH": "", "TB_WALK_CYCLE_OPEN_TARGET": "", "TB_WALK_CYCLE_OPEN_WEIGHT": "", "TB_WALK_SUBSPACE_RANK": "0", "TB_WALK_HIER_EVIDENCE": "0", "TB_WALK_EVIDENCE": "0", "TB_WALK_DATA_TRAJ": "0", "TB_WALK_HIER_K": "", "TB_WALK_HIER_TRAJ": "", "TB_WALK_HIER_CHANNEL_GATE": "", "TB_WALK_HIER_LEARNED": "", "TB_WALK_HIER_AFFINE_FREE": "", "TB_WALK_HIER_DEPTH_RMS": "", "TB_WALK_HIER_ARCHIVE": "", "TB_WALK_HIER_ARCHIVE_CHUNK": "", "TB_WALK_HIER_ARCHIVE_TOPK": "", "TB_WALK_PRED": "0", "TB_WALK_PRED_CE": "0", "TB_WALK_SUBSPACE_GATED": "0", "TB_WALK_SUBSPACE_CONTENT": "0", "TB_WALK_MEMORY_CHUNK": "0", "TB_WALK_PATH_MIXER": "0", "TB_WALK_DUAL_PATH_MIXER": "0", "TB_TIE_EMB_HEAD": "0", "TB_POS_EMB": "1", "TB_MEM": "0", "TB_NI": "0", "TB_NI_MODE": "", "TB_NI_STEPS": "", "TB_NI_RANK": "", "TB_NI_WIDTH_MULT": "", "TB_NI_LAYERS": "", "TB_PHASE_B_SUPERSET": "1", "TB_PHASE_WALK_LAYERS": "all", "TB_PHASE_WALK_RANK": "32", "TB_PHASE_WALK_K": "2", "TB_PHASE_WALK_DATA_TRAJ": "1", "TB_PHASE_WALK_TRAJ": "full", "TB_PHASE_HIER_SCALES": "8,64", "TB_PHASE_WALK_NORM": "0.05", "TB_PHASE_WALK_NORM_MODE": "clip", "TB_PHASE_ARCHIVE_CHUNK": "64", "TB_PHASE_ARCHIVE_TOPK": "1", "TB_TRAJ_REG": "0", "TB_TRAJ_REG_RANK": "", "TB_TRAJ_REG_HORIZON": "", "TB_KMERGE": "0", "TB_KMERGE_K": "", "TB_KMERGE_PREFIX_FRAC": "", "TB_LEXICAL_SEMANTIC": "0", "TB_LEXICAL_SEMANTIC_RANK": "256", "TB_SEMANTIC_WD": "1", "TB_V41_ENGRAM": "0", "TB_V41_ENGRAM_BUCKETS": "", "TB_V41_ENGRAM_RANK": "", "TB_V41_KV_FP4": "0", "TB_V41_CED": "0", "TB_V41_CSA2": "0", "TB_PAUSE_N": "", "TB_PAUSE_EVERY": "", "TB_PAUSE_HUGINN": "", "TB_PAUSE_RANK": "", "TB_R_ENABLE": "0", "TB_R_START": "", "TB_R_END": "", "TB_R_MEAN": "", "TB_R_MAX": "", "TB_R_BACKPROP": "", "TB_R_INSTR_ONLY": "", "TB_R_KV_SHARE": "", "TB_R_UPDATE": "", "TB_R_GAIN": "", "TB_WALK_ADAPT_SCALE": "", "TB_WALK_ADAPT_LAGS": "", "TB_WALK_ADAPT_RANK": "", "TB_WALK_CONV_BASELINE": "", "TB_LOOP_ENABLE": "0", "TB_LOOP_START": "", "TB_LOOP_END": "", "TB_LOOP_PASSES": ""}}, "data": {"version": 4, "env": {"TB_STREAM": "", "TB_STREAM_REVISION": "", "TB_STREAM_ND": "", "TB_STREAM_DOMAIN": "", "TB_STREAM_TEXT_FIELD": "", "TB_STREAM_PRESET": "", "TB_STREAM_MIX": "[{\"budget_tokens\":750000000,\"dataset\":\"HuggingFaceTB/smollm-corpus:cosmopedia-v2\",\"domain_key\":\"prompt\",\"domains\":3,\"name\":\"cosmopedia_v2\",\"revision\":\"3ba9d60\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":4500000000,\"dataset\":\"wikimedia/wikipedia:20231101.en\",\"domain_key\":\"title\",\"domains\":4,\"name\":\"wikipedia_en\",\"revision\":\"b04c8d1ceb2f5cd4588862100d08de323dccfbaa\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":2550000000,\"dataset\":\"OctoThinker/MegaMath-Web-Pro-Max\",\"domain_key\":\"id\",\"domains\":4,\"name\":\"megamath_web_pro_max\",\"revision\":\"0194035ca21e11ce60c4693fd2607e621dec9324\",\"text_field\":\"text\",\"weight\":0.1},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-RQA\",\"name\":\"stem_rqa\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1.2\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Fact-Seeking\",\"name\":\"fact_seeking\",\"revision\":\"807afc1fa65c441d46ebc7d9b95295a35499a527\",\"text_field\":\"text\",\"weight\":0.05},{\"budget_tokens\":600000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-InfiniByte-Reasoning\",\"name\":\"infinibyte\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.04},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Math-Textbooks\",\"name\":\"math_textbooks\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.06},{\"budget_tokens\":1800000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:vendor\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_vendor\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Math-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data\",\"message_field\":\"messages\",\"name\":\"sft_math\",\"revision\":\"8e793210e175b6406c752a870f585f62de98c0d3\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.09},{\"budget_tokens\":380000000,\"dataset\":\"HuggingFaceTB/smoltalk\",\"dedup\":\"first_user\",\"domain_key\":\"category\",\"domains\":3,\"file_selector\":\"smol-magpie-ultra/train\",\"filters\":{\"category\":[\"not_in\",[\"coding\",\"math\"]],\"quality\":[\"not_in\",[\"poor\",\"very poor\"]]},\"message_field\":\"messages\",\"name\":\"sft_chat\",\"revision\":\"5feaf2fd3ffca7c237fc38d1861bc30365d48ffa\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":2100000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-Wiki-Rewrite\",\"name\":\"wiki_rewrite\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.14},{\"budget_tokens\":1200000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"domain_key\":\"uuid\",\"domains\":2,\"file_selector\":\"Nemotron-Pretraining-Scientific-Coding\",\"name\":\"scientific_coding\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":900000000,\"dataset\":\"nvidia/Nemotron-SFT-Instruction-Following-Chat-v2\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"data/reasoning_off\",\"message_field\":\"messages\",\"name\":\"sft_if_chat\",\"revision\":\"1a9454ed054b8544503ab8d8c0a519d141a44c5b\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.08},{\"budget_tokens\":700000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:rqa\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_rqa\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":160000000,\"dataset\":\"nvidia/Nemotron-SFT-Science-v2:so\",\"dedup\":\"first_user\",\"domain_key\":\"uuid\",\"domains\":3,\"message_field\":\"messages\",\"name\":\"sft_science_so\",\"revision\":\"6536a5021222a94126968e8c92f29ee47fc8a7df\",\"template\":\"chat-v1\",\"text_field\":\"text\",\"weight\":0.0},{\"budget_tokens\":1500000000,\"dataset\":\"nvidia/Nemotron-Pretraining-Specialized-v1\",\"dedup\":\"text_head\",\"domain_key\":\"uuid\",\"domains\":3,\"file_selector\":\"Nemotron-Pretraining-STEM-SFT\",\"name\":\"stem_sft\",\"revision\":\"ff05e40d9129acd9fd29335cf01b2b5e16bda374\",\"text_field\":\"text\",\"weight\":0.1}]", "TB_STREAM_VAL_SOURCE": "", "TB_STREAM_SHUFFLE_BUFFER": "0", "TB_STREAM_SHUFFLE_SEED": "0", "TB_CODE_STREAM": "", "TB_CODE_STREAM_REVISION": "", "TB_CODE_DATA_DIR": "", "TB_CODE_STREAM_ND": "", "TB_CODE_TEXT_FIELD": "", "TB_CODE_REPO_FIELD": "", "TB_CODE_PATH_FIELD": "", "TB_CODE_LANG_FIELD": "", "TB_CODE_GROUP_REPO": "", "TB_CODE_LANG_FILTER": "", "TB_CODE_SHARE": "", "TB_CODE_SHARE_START": "", "TB_CODE_RAMP_TOKENS": "", "TB_CODE_VAL_TOKENS": "", "TB_CODE_VAL_PATH": "", "TB_COT_URL": "", "TB_COT_PATH": "", "TB_SFT_MASK": "1"}, "stream_mix_signature": "632bd873d5e4296a", "tokenizer": "gpt2", "vocab": 50257, "packing": {"fineweb": "eos-v1", "code": "eos-v1", "windowing": "contiguous-no-drop-v2"}}, "runtime": {"TB_COMPILE": "0", "TB_COMPILE_DYNAMIC": "0", "TB_ACTIVATION_CKPT_STRIDE": "1", "TB_ACTIVATION_CKPT_FULL_AT_T": "0", "TB_WALK_LOOP_CKPT": "0", "TB_ATTN_WINDOW_CHUNK": "512", "TB_PHASE_ASSOC_CHUNK": "0", "TB_WALK_ASSOC_CHUNK": "0", "TB_LOCAL_CONV_FUSE_KERNELS": "1", "TB_WALK_MIX_TENSORCORE": "0", "TB_METRIC_EVERY": "1", "TB_LOG_EVERY": "250", "TB_TREND_EVERY": "50", "TB_CKPT_EVERY": "250", "TB_BASELINE_VAL_ON_RESUME": "1", "TB_WALK_ABLATE_EVAL": "0", "TB_PHASE_ABLATE_EVAL": "0", "TB_V41_ABLATE_EVAL": "0", "TB_GRAD_CLIP": "2.0", "TB_ADAPTER_MULT": "1.0", "TB_PHASE_MULT": "1.0", "TB_LOCAL_CONV_MULT": "1.0", "TB_LEXICAL_SEMANTIC_MULT": "1.0", "TB_LEXICAL_SEMANTIC_AUX_WEIGHT": "0.05", "TB_LEXICAL_SEMANTIC_STRIDE": "32", "TB_LEXICAL_SEMANTIC_SCALE": "1.0", "TB_LEXICAL_SEMANTIC_TEMPERATURE": "0.1", "TB_WALK_FORWARD_SCALE": "1.0", "TB_PHASE_WALK_FORWARD_SCALE": "0.0", "TB_WALK_INPUT_GRAD_SCALE": "1.0", "TB_PHASE_WALK_INPUT_GRAD_SCALE": "1.0", "TB_PHASE_PRED_WEIGHT": "0.0", "TB_PHASE_PRED_SCALE": "0.0", "TB_R_RESID_LOG": "", "TB_R_HALT_EPS": "", "TB_R_HALT_MIN": "", "TB_R_ADAPTER_MULT": "", "TB_EMA_EVAL": "1", "TB_EMA_DECAY": "0.999", "TB_OPT_KAHAN": "1", "TB_R_SAMPLE": "", "TB_LR_COOLDOWN_START": "47750", "TB_LR_COOLDOWN_FLOOR": "2e-05", "TB_BOTTOM_ISOLATE": "1", "TB_BOTTOM_ISOLATE_RAMP": "500", "TB_BOTTOM_LR_MULT": "0.25", "TB_HF_CKPT_BRANCH": "47750"}}}