| { |
| "architecture_class": "SolForCausalLM", |
| "attention": "Triton fused causal GQA via compiled FlexAttention; XSA and SDPA disabled", |
| "attention_backend": "triton", |
| "base_architecture": "Sol Lite", |
| "batch_mixing": "all scheduled source quotas are assembled in every optimizer update, then shuffled together", |
| "compile": true, |
| "compile_mode": "reduce-overhead", |
| "compiler_backend": "TorchInductor/Triton", |
| "cpu_threads": 16, |
| "data_sources": { |
| "arithmetic_generated": "on-the-fly deterministic arithmetic with worked answers", |
| "code": "CoRNStack Python query/code pairs", |
| "finemath_4p5plus": "FineMath score >=4.5 heldout split", |
| "finemath_4plus": "FineMath 4+ base split", |
| "fineweb_edu_3p5plus": "FineWeb-Edu score >=3.5 heldout split", |
| "general": "FineWeb-Edu score >=3; stable heldout split; science topics routed separately", |
| "openwebmath": "OpenWebMath", |
| "physical_science": "FineWeb-Edu score >=3 with science-topic filter", |
| "procedural": "Cosmopedia-v2 instructional/procedural subset" |
| }, |
| "decay_start_update_index": 17166, |
| "decay_updates": 1908, |
| "fineweb_edu_source_pool": "sample-10BT; direct bounded Parquet streaming", |
| "flash_attention_4_version": null, |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "gpu_compute_capability": [ |
| 12, |
| 0 |
| ], |
| "gpu_memory_bytes": 101975851008, |
| "gradient_accumulation": 1, |
| "input_pipeline": "JIT streaming from public dataset sources; CPU fetch/filter/tokenization overlaps GPU training", |
| "microbatch_sequences": 512, |
| "mix": { |
| "corrective": { |
| "arithmetic_generated": 8.0, |
| "code": 4.0, |
| "finemath_4plus": 20.0, |
| "general": 45.0, |
| "openwebmath": 12.0, |
| "physical_science": 3.0, |
| "procedural": 8.0 |
| }, |
| "high_quality": { |
| "arithmetic_generated": 10.0, |
| "code": 4.0, |
| "finemath_4p5plus": 30.0, |
| "fineweb_edu_3p5plus": 30.0, |
| "openwebmath": 20.0, |
| "physical_science": 2.0, |
| "procedural": 4.0 |
| }, |
| "legacy": { |
| "arithmetic_generated": 3.0, |
| "code": 4.0, |
| "finemath_4plus": 7.5, |
| "general": 65.0, |
| "openwebmath": 4.5, |
| "physical_science": 4.0, |
| "procedural": 12.0 |
| } |
| }, |
| "mix_boundaries_tokens": { |
| "corrective_ramp_end": 1400111104, |
| "high_quality_start": 4499963904, |
| "legacy_end": 1333264384 |
| }, |
| "model": "Sol Nano", |
| "optimizer": "PyTorch fused AdamW", |
| "parameters": 2895188, |
| "peak_learning_rate": 0.001, |
| "schedule": "2% linear warmup, stable through 90%, final 10% linear decay to zero", |
| "seed": 20260929, |
| "sequence_length": 512, |
| "sequences_per_update": 512, |
| "target_tokens": 5000000000, |
| "tokenizer_sha256": "72a97c4c592c991733fcc6ad687734ce51e1ba25b0f7c2faf611ac30edc50231", |
| "tokens_per_update": 262144, |
| "total_updates": 19074, |
| "triton_version": "3.6.0", |
| "use_xsa": false, |
| "variant": "sol_nano_2p9m_tn_gram", |
| "warmup_updates": 382 |
| } |
|
|
