| { |
| "cache_key": { |
| "version": 1, |
| "kind": "gpu", |
| "gpu": { |
| "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "total_memory": 101975851008, |
| "capability": [ |
| 12, |
| 0 |
| ] |
| }, |
| "torch": "2.8.0+cu128", |
| "cuda": "12.8", |
| "python": "3.13.11", |
| "packages": { |
| "transformers": "4.57.1", |
| "numpy": "2.2.6", |
| "pyarrow": "21.0.0", |
| "huggingface_hub": "0.36.0" |
| }, |
| "settings": { |
| "context": 2048, |
| "global_batch": 90, |
| "microbatch_candidates": [ |
| 8, |
| 16, |
| 32, |
| 48, |
| 64, |
| 72, |
| 76, |
| 80, |
| 86, |
| 90 |
| ], |
| "batch_selection": "max_batch", |
| "gradient_checkpointing": false, |
| "compile_model": true, |
| "autotune_microbatch": true, |
| "attention_backend": "flash", |
| "vram_fraction": 0.98 |
| }, |
| "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b", |
| "requested_microbatch_ceiling": 90 |
| }, |
| "cache_reused": true, |
| "attention": "PyTorch FlashAttention (forced; forward/backward verified)", |
| "selected": { |
| "microbatch": 90, |
| "mode": "compiled", |
| "microbatch_seconds": 0.9387972919998901, |
| "peak_gb": 89.37766551971436, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9387972919998901, |
| "input_tokens_per_second": 196336.31410178967 |
| }, |
| "measurements": [ |
| { |
| "microbatch": 8, |
| "mode": "eager", |
| "microbatch_seconds": 0.15844926500000156, |
| "peak_gb": 14.764304637908936, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.8007989265000077, |
| "input_tokens_per_second": 103402.18365796673 |
| }, |
| { |
| "microbatch": 16, |
| "mode": "eager", |
| "microbatch_seconds": 0.34631374599999276, |
| "peak_gb": 28.084960460662842, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.929804849999968, |
| "input_tokens_per_second": 94619.40329680326 |
| }, |
| { |
| "microbatch": 32, |
| "mode": "eager", |
| "microbatch_seconds": 0.7831460159999892, |
| "peak_gb": 54.827155113220215, |
| "fits_budget": true, |
| "estimated_update_seconds": 2.188403235499976, |
| "input_tokens_per_second": 83682.98971210101 |
| }, |
| { |
| "microbatch": 48, |
| "mode": "eager", |
| "microbatch_seconds": 1.1706683780000162, |
| "peak_gb": 81.63183498382568, |
| "fits_budget": true, |
| "estimated_update_seconds": 2.1949380675000043, |
| "input_tokens_per_second": 83972.54239321277 |
| }, |
| { |
| "microbatch": 8, |
| "mode": "compiled", |
| "microbatch_seconds": 0.08827481200000875, |
| "peak_gb": 9.113307476043701, |
| "fits_budget": true, |
| "estimated_update_seconds": 1.008851873500106, |
| "input_tokens_per_second": 185602.20779624404 |
| }, |
| { |
| "microbatch": 16, |
| "mode": "compiled", |
| "microbatch_seconds": 0.17047579999996287, |
| "peak_gb": 16.87287473678589, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9571819019997747, |
| "input_tokens_per_second": 192214.96540862185 |
| }, |
| { |
| "microbatch": 32, |
| "mode": "compiled", |
| "microbatch_seconds": 0.34263174000000163, |
| "peak_gb": 32.49641752243042, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9525396829999977, |
| "input_tokens_per_second": 191272.41393339593 |
| }, |
| { |
| "microbatch": 48, |
| "mode": "compiled", |
| "microbatch_seconds": 0.5075005140000144, |
| "peak_gb": 48.123157024383545, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9488392155000156, |
| "input_tokens_per_second": 193702.2668709991 |
| }, |
| { |
| "microbatch": 64, |
| "mode": "compiled", |
| "microbatch_seconds": 0.6643874849999634, |
| "peak_gb": 63.757601737976074, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.926719353999971, |
| "input_tokens_per_second": 197282.46386219515 |
| }, |
| { |
| "microbatch": 72, |
| "mode": "compiled", |
| "microbatch_seconds": 0.7446585950000326, |
| "peak_gb": 71.65071392059326, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9283865335000314, |
| "input_tokens_per_second": 198018.26097232322 |
| }, |
| { |
| "microbatch": 76, |
| "mode": "compiled", |
| "microbatch_seconds": 0.7858128550000174, |
| "peak_gb": 75.5242338180542, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9300540555000225, |
| "input_tokens_per_second": 198072.60597689834 |
| }, |
| { |
| "microbatch": 80, |
| "mode": "compiled", |
| "microbatch_seconds": 0.8269688110000288, |
| "peak_gb": 79.40062236785889, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9293605420000404, |
| "input_tokens_per_second": 198121.13567121493 |
| }, |
| { |
| "microbatch": 86, |
| "mode": "compiled", |
| "microbatch_seconds": 0.8882591199999865, |
| "peak_gb": 85.4687147140503, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9413692904999778, |
| "input_tokens_per_second": 198284.4825730612 |
| }, |
| { |
| "microbatch": 90, |
| "mode": "compiled", |
| "microbatch_seconds": 0.9302335780000135, |
| "peak_gb": 89.3772840499878, |
| "fits_budget": true, |
| "estimated_update_seconds": 0.9302335780000135, |
| "input_tokens_per_second": 198143.7827650609 |
| } |
| ], |
| "failures": [ |
| { |
| "microbatch": 64, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 72, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 76, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 80, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 86, |
| "mode": "eager", |
| "reason": "oom" |
| }, |
| { |
| "microbatch": 90, |
| "mode": "eager", |
| "reason": "oom" |
| } |
| ], |
| "selection_policy": "max_batch", |
| "budget_gb": 92.52170804977416, |
| "reserve_gb": 2.1791439056396484, |
| "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time.", |
| "legacy_seed_verified": false |
| } |
