File size: 2,934 Bytes
398cfcf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | {
"architecture_class": "SolForCausalLM",
"attention": "Triton fused causal GQA via compiled FlexAttention; XSA and SDPA disabled",
"attention_backend": "triton",
"base_architecture": "Sol Lite",
"batch_mixing": "all scheduled source quotas are assembled in every optimizer update, then shuffled together",
"compile": true,
"compile_mode": "reduce-overhead",
"compiler_backend": "TorchInductor/Triton",
"cpu_threads": 16,
"data_sources": {
"arithmetic_generated": "on-the-fly deterministic arithmetic with worked answers",
"code": "CoRNStack Python query/code pairs",
"finemath_4p5plus": "FineMath score >=4.5 heldout split",
"finemath_4plus": "FineMath 4+ base split",
"fineweb_edu_3p5plus": "FineWeb-Edu score >=3.5 heldout split",
"general": "FineWeb-Edu score >=3; stable heldout split; science topics routed separately",
"openwebmath": "OpenWebMath",
"physical_science": "FineWeb-Edu score >=3 with science-topic filter",
"procedural": "Cosmopedia-v2 instructional/procedural subset"
},
"decay_start_update_index": 17166,
"decay_updates": 1908,
"fineweb_edu_source_pool": "sample-10BT; direct bounded Parquet streaming",
"flash_attention_4_version": null,
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
"gpu_compute_capability": [
12,
0
],
"gpu_memory_bytes": 101975851008,
"gradient_accumulation": 1,
"input_pipeline": "JIT streaming from public dataset sources; CPU fetch/filter/tokenization overlaps GPU training",
"microbatch_sequences": 512,
"mix": {
"corrective": {
"arithmetic_generated": 8.0,
"code": 4.0,
"finemath_4plus": 20.0,
"general": 45.0,
"openwebmath": 12.0,
"physical_science": 3.0,
"procedural": 8.0
},
"high_quality": {
"arithmetic_generated": 10.0,
"code": 4.0,
"finemath_4p5plus": 30.0,
"fineweb_edu_3p5plus": 30.0,
"openwebmath": 20.0,
"physical_science": 2.0,
"procedural": 4.0
},
"legacy": {
"arithmetic_generated": 3.0,
"code": 4.0,
"finemath_4plus": 7.5,
"general": 65.0,
"openwebmath": 4.5,
"physical_science": 4.0,
"procedural": 12.0
}
},
"mix_boundaries_tokens": {
"corrective_ramp_end": 1400111104,
"high_quality_start": 4499963904,
"legacy_end": 1333264384
},
"model": "Sol Nano",
"optimizer": "PyTorch fused AdamW",
"parameters": 2895188,
"peak_learning_rate": 0.001,
"schedule": "2% linear warmup, stable through 90%, final 10% linear decay to zero",
"seed": 20260929,
"sequence_length": 512,
"sequences_per_update": 512,
"target_tokens": 5000000000,
"tokenizer_sha256": "72a97c4c592c991733fcc6ad687734ce51e1ba25b0f7c2faf611ac30edc50231",
"tokens_per_update": 262144,
"total_updates": 19074,
"triton_version": "3.6.0",
"use_xsa": false,
"variant": "sol_nano_2p9m_tn_gram",
"warmup_updates": 382
}
|