File size: 1,369 Bytes
11efa64 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | [
{
"optimizer": "adamw",
"category": "1. AdamW",
"fwd_bwd_ms": 44.51255885263284,
"step_ms": 3.5697922110557556,
"hessian_amortised_ms": 0.0,
"total_ms": 48.0823510636886,
"state_mb": 216.584192,
"state_multiple": 2.0
},
{
"optimizer": "mars",
"category": "2. Variance-reduced AdamW variant",
"fwd_bwd_ms": 43.19541243215402,
"step_ms": 6.120803082982699,
"hessian_amortised_ms": 0.0,
"total_ms": 49.31621551513672,
"state_mb": 324.876288,
"state_multiple": 3.0
},
{
"optimizer": "lion",
"category": "3. Memory-efficient",
"fwd_bwd_ms": 44.26518057783445,
"step_ms": 3.246958553791046,
"hessian_amortised_ms": 0.0,
"total_ms": 47.512139131625496,
"state_mb": 108.292096,
"state_multiple": 1.0
},
{
"optimizer": "muon",
"category": "4. Matrix-based",
"fwd_bwd_ms": 41.5445347627004,
"step_ms": 12.41101324558258,
"hessian_amortised_ms": 0.0,
"total_ms": 53.95554800828298,
"state_mb": 141.08672,
"state_multiple": 1.3028348809501296
},
{
"optimizer": "sophia",
"category": "5. Hessian-based (bonus)",
"fwd_bwd_ms": 44.653950879971184,
"step_ms": 2.7560430268446603,
"hessian_amortised_ms": 4.465395087997118,
"total_ms": 51.87538899481296,
"state_mb": 216.584192,
"state_multiple": 2.0
}
] |