tierb-b200-gbs8 / granularity_next_b200 /memory_summary_all.json
AbhishekGhosh1998's picture
Upload granularity_next_b200/memory_summary_all.json with huggingface_hub
ab7d26c verified
Raw History Blame Contribute Delete
64.2 kB
{ "runs": [ { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1", "config": "DEEPSEEK_V3_L4", "backend": "echo", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.13671875", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 68.58609199523926, "max_reserved_gib": 82.048828125, "fragmentation_gib": 13.462736129760742, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 128.0029296875, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 128.0029296875 }, { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "DEEPSEEK_V3_L4", "backend": "echo", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 69.96967601776123, "max_reserved_gib": 79.791015625, "fragmentation_gib": 9.82133960723877, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 116.5556640625, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 116.5556640625 }, { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1", "config": "DEEPSEEK_V3_L4", "backend": "moonep", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 56.274383544921875, "max_reserved_gib": 59.23046875, "fragmentation_gib": 2.956085205078125, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 95.5556640625, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 95.5556640625 }, { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1", "config": "DEEPSEEK_V3_L4", "backend": "moonep", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 56.25963544845581, "max_reserved_gib": 59.21484375, "fragmentation_gib": 2.9552083015441895, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 81.8642578125, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 81.8642578125 }, { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1", "config": "DEEPSEEK_V3_L4", "backend": "ultraep", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.13671875", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 64.16005086898804, "max_reserved_gib": 66.474609375, "fragmentation_gib": 2.314558506011963, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 99.6044921875, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 99.6044921875 }, { "run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/ultraep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "DEEPSEEK_V3_L4", "backend": "ultraep", "num_layers": "4", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 64.25602293014526, "max_reserved_gib": 66.458984375, "fragmentation_gib": 2.2029614448547363, "theoretical_weight_optim_gib": 97.629130859375, "peak_sampled_gib": 100.2666015625, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 100.2666015625 }, { "run_dir": "tiera_topk_memory/G2_K2/echo/tierbmem/tierbmem_gbs16/G2_K2__echo__matched__seq4096__gbs16__cap1p15625__memiteration__it6/rep1", "config": "G2_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.15625", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 74.99560832977295, "max_reserved_gib": 88.99609375, "fragmentation_gib": 14.00048542022705, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 88.99609375 }, { "run_dir": "tiera_topk_memory/G2_K2/echo/tierbmem/tierbmem_gbs16/G2_K2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G2_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 69.40528774261475, "max_reserved_gib": 85.388671875, "fragmentation_gib": 15.983384132385254, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 32, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 85.388671875 }, { "run_dir": "tiera_topk_memory/G2_K2/moonep/tierbmem/tierbmem_gbs16/G2_K2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G2_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.84062576293945, "max_reserved_gib": 56.009765625, "fragmentation_gib": 4.169139862060547, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 56.009765625 }, { "run_dir": "tiera_topk_memory/G2_K2/moonep/tierbmem/tierbmem_gbs16/G2_K2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G2_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.80883169174194, "max_reserved_gib": 55.9375, "fragmentation_gib": 4.128668308258057, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 55.9375 }, { "run_dir": "tiera_topk_memory/G2_K2/ultraep/tierbmem/tierbmem_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625__memiteration__it6/rep1", "config": "G2_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.15625", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 64.17789077758789, "max_reserved_gib": 77.595703125, "fragmentation_gib": 13.41781234741211, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 77.595703125 }, { "run_dir": "tiera_topk_memory/G2_K2/ultraep/tierbmem/tierbmem_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G2_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 63.95792531967163, "max_reserved_gib": 78.193359375, "fragmentation_gib": 14.23543405532837, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 78.193359375 }, { "run_dir": "tiera_topk_memory/G3_K2/echo/tierbmem/tierbmem_gbs16/G3_K2__echo__matched__seq4096__gbs16__cap1p1875__memiteration__it6/rep1", "config": "G3_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.1875", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 74.04066181182861, "max_reserved_gib": 90.314453125, "fragmentation_gib": 16.273791313171387, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 90.314453125 }, { "run_dir": "tiera_topk_memory/G3_K2/echo/tierbmem/tierbmem_gbs16/G3_K2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G3_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 68.66449165344238, "max_reserved_gib": 85.119140625, "fragmentation_gib": 16.454648971557617, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 85.119140625 }, { "run_dir": "tiera_topk_memory/G3_K2/moonep/tierbmem/tierbmem_gbs16/G3_K2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G3_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.82174253463745, "max_reserved_gib": 54.93359375, "fragmentation_gib": 3.111851215362549, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 54.93359375 }, { "run_dir": "tiera_topk_memory/G3_K2/moonep/tierbmem/tierbmem_gbs16/G3_K2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G3_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.79027843475342, "max_reserved_gib": 54.931640625, "fragmentation_gib": 3.141362190246582, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 54.931640625 }, { "run_dir": "tiera_topk_memory/G3_K2/ultraep/tierbmem/tierbmem_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875__memiteration__it6/rep1", "config": "G3_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.1875", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 62.94294357299805, "max_reserved_gib": 78.802734375, "fragmentation_gib": 15.859790802001953, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 78.802734375 }, { "run_dir": "tiera_topk_memory/G3_K2/ultraep/tierbmem/tierbmem_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G3_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 62.87725257873535, "max_reserved_gib": 78.611328125, "fragmentation_gib": 15.734075546264648, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 78.611328125 }, { "run_dir": "tiera_topk_memory/G3_K8/echo/tierbmem/tierbmem_gbs16/G3_K8__echo__matched__seq4096__gbs16__cap1p09375__memiteration__it6/rep1", "config": "G3_K8", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.09375", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 79.24201774597168, "max_reserved_gib": 86.02734375, "fragmentation_gib": 6.78532600402832, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 86.02734375 }, { "run_dir": "tiera_topk_memory/G3_K8/echo/tierbmem/tierbmem_gbs16/G3_K8__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G3_K8", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 73.50973176956177, "max_reserved_gib": 80.931640625, "fragmentation_gib": 7.421908855438232, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 80.931640625 }, { "run_dir": "tiera_topk_memory/G3_K8/moonep/tierbmem/tierbmem_gbs16/G3_K8__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G3_K8", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.8156418800354, "max_reserved_gib": 54.701171875, "fragmentation_gib": 2.8855299949645996, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 54.701171875 }, { "run_dir": "tiera_topk_memory/G3_K8/moonep/tierbmem/tierbmem_gbs16/G3_K8__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G3_K8", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.79296684265137, "max_reserved_gib": 54.62890625, "fragmentation_gib": 2.835939407348633, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 54.62890625 }, { "run_dir": "tiera_topk_memory/G3_K8/ultraep/tierbmem/tierbmem_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375__memiteration__it6/rep1", "config": "G3_K8", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.09375", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 68.15336608886719, "max_reserved_gib": 74.333984375, "fragmentation_gib": 6.1806182861328125, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 74.333984375 }, { "run_dir": "tiera_topk_memory/G3_K8/ultraep/tierbmem/tierbmem_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G3_K8", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 67.73253440856934, "max_reserved_gib": 72.55859375, "fragmentation_gib": 4.826059341430664, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 72.55859375 }, { "run_dir": "tiera_topk_memory/G4_K16/echo/tierbmem/tierbmem_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1", "config": "G4_K16", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.08203125", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 82.14846229553223, "max_reserved_gib": 85.353515625, "fragmentation_gib": 3.2050533294677734, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 85.353515625 }, { "run_dir": "tiera_topk_memory/G4_K16/echo/tierbmem/tierbmem_gbs16/G4_K16__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K16", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 76.34746932983398, "max_reserved_gib": 80.279296875, "fragmentation_gib": 3.9318275451660156, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 32, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 80.279296875 }, { "run_dir": "tiera_topk_memory/G4_K16/moonep/tierbmem/tierbmem_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G4_K16", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.87144136428833, "max_reserved_gib": 53.533203125, "fragmentation_gib": 1.66176176071167, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 53.533203125 }, { "run_dir": "tiera_topk_memory/G4_K16/moonep/tierbmem/tierbmem_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G4_K16", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.8409538269043, "max_reserved_gib": 53.5078125, "fragmentation_gib": 1.6668586730957031, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 53.5078125 }, { "run_dir": "tiera_topk_memory/G4_K16/ultraep/tierbmem/tierbmem_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1", "config": "G4_K16", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.08203125", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 70.50301837921143, "max_reserved_gib": 73.294921875, "fragmentation_gib": 2.791903495788574, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 73.294921875 }, { "run_dir": "tiera_topk_memory/G4_K16/ultraep/tierbmem/tierbmem_gbs16/G4_K16__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K16", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 69.96756029129028, "max_reserved_gib": 72.544921875, "fragmentation_gib": 2.577361583709717, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 72.544921875 }, { "run_dir": "tiera_topk_memory/G4_K2/echo/tierbmem/tierbmem_gbs16/G4_K2__echo__matched__seq4096__gbs16__cap1p265625__memiteration__it6/rep1", "config": "G4_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.265625", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 74.2385311126709, "max_reserved_gib": 83.779296875, "fragmentation_gib": 9.540765762329102, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 83.779296875 }, { "run_dir": "tiera_topk_memory/G4_K2/echo/tierbmem/tierbmem_gbs16/G4_K2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K2", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 68.84133005142212, "max_reserved_gib": 78.705078125, "fragmentation_gib": 9.86374807357788, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 78.705078125 }, { "run_dir": "tiera_topk_memory/G4_K2/moonep/tierbmem/tierbmem_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G4_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.8655686378479, "max_reserved_gib": 54.099609375, "fragmentation_gib": 2.2340407371520996, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 54.099609375 }, { "run_dir": "tiera_topk_memory/G4_K2/moonep/tierbmem/tierbmem_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G4_K2", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.83556938171387, "max_reserved_gib": 54.07421875, "fragmentation_gib": 2.238649368286133, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 54.07421875 }, { "run_dir": "tiera_topk_memory/G4_K2/ultraep/tierbmem/tierbmem_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625__memiteration__it6/rep1", "config": "G4_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.265625", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 62.593889236450195, "max_reserved_gib": 71.72265625, "fragmentation_gib": 9.128767013549805, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 71.72265625 }, { "run_dir": "tiera_topk_memory/G4_K2/ultraep/tierbmem/tierbmem_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K2", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 62.47251892089844, "max_reserved_gib": 71.435546875, "fragmentation_gib": 8.963027954101562, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 71.435546875 }, { "run_dir": "tiera_topk_memory/G4_K4/echo/tierbmem/tierbmem_gbs16/G4_K4__echo__matched__seq4096__gbs16__cap1p171875__memiteration__it6/rep1", "config": "G4_K4", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.171875", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 75.35982322692871, "max_reserved_gib": 82.447265625, "fragmentation_gib": 7.087442398071289, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 82.447265625 }, { "run_dir": "tiera_topk_memory/G4_K4/echo/tierbmem/tierbmem_gbs16/G4_K4__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K4", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 69.92792558670044, "max_reserved_gib": 77.9765625, "fragmentation_gib": 8.04863691329956, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 77.9765625 }, { "run_dir": "tiera_topk_memory/G4_K4/moonep/tierbmem/tierbmem_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G4_K4", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.86496019363403, "max_reserved_gib": 53.697265625, "fragmentation_gib": 1.8323054313659668, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 53.697265625 }, { "run_dir": "tiera_topk_memory/G4_K4/moonep/tierbmem/tierbmem_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G4_K4", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.834007263183594, "max_reserved_gib": 53.626953125, "fragmentation_gib": 1.7929458618164062, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 53.626953125 }, { "run_dir": "tiera_topk_memory/G4_K4/ultraep/tierbmem/tierbmem_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875__memiteration__it6/rep1", "config": "G4_K4", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.171875", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 63.71808052062988, "max_reserved_gib": 70.310546875, "fragmentation_gib": 6.592466354370117, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 70.310546875 }, { "run_dir": "tiera_topk_memory/G4_K4/ultraep/tierbmem/tierbmem_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K4", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 63.531575202941895, "max_reserved_gib": 70.23828125, "fragmentation_gib": 6.7067060470581055, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 70.23828125 }, { "run_dir": "tiera_topk_memory/G4_K8/echo/tierbmem/tierbmem_gbs16/G4_K8__echo__matched__seq4096__gbs16__cap1p109375__memiteration__it6/rep1", "config": "G4_K8", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.109375", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 77.58307456970215, "max_reserved_gib": 80.9453125, "fragmentation_gib": 3.3622379302978516, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 80.9453125 }, { "run_dir": "tiera_topk_memory/G4_K8/echo/tierbmem/tierbmem_gbs16/G4_K8__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K8", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 72.07384824752808, "max_reserved_gib": 75.841796875, "fragmentation_gib": 3.767948627471924, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 75.841796875 }, { "run_dir": "tiera_topk_memory/G4_K8/moonep/tierbmem/tierbmem_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1", "config": "G4_K8", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.8758282661438, "max_reserved_gib": 53.712890625, "fragmentation_gib": 1.8370623588562012, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 43, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 53.712890625 }, { "run_dir": "tiera_topk_memory/G4_K8/moonep/tierbmem/tierbmem_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1", "config": "G4_K8", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 51.846561431884766, "max_reserved_gib": 53.701171875, "fragmentation_gib": 1.8546104431152344, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 53.701171875 }, { "run_dir": "tiera_topk_memory/G4_K8/ultraep/tierbmem/tierbmem_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__cap1p109375__memiteration__it6/rep1", "config": "G4_K8", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.109375", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 65.95306587219238, "max_reserved_gib": 68.87109375, "fragmentation_gib": 2.918027877807617, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 0, "effective_peak_gib": 68.87109375 }, { "run_dir": "tiera_topk_memory/G4_K8/ultraep/tierbmem/tierbmem_gbs16/G4_K8__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1", "config": "G4_K8", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "16", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 65.67387962341309, "max_reserved_gib": 68.630859375, "fragmentation_gib": 2.956979751586914, "theoretical_weight_optim_gib": 165.067802734375, "snapshot_pickle_count": 0, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 0, "effective_peak_gib": 68.630859375 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/echo/tierbmem/tierbmem_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1", "config": "KIMI_K2_L2", "backend": "echo", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.17578125", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 56.259039878845215, "max_reserved_gib": 72.916015625, "fragmentation_gib": 16.656975746154785, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 129.1201171875, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 129.1201171875 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/echo/tierbmem/tierbmem_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "KIMI_K2_L2", "backend": "echo", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 58.91231727600098, "max_reserved_gib": 71.5078125, "fragmentation_gib": 12.595495223999023, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 116.1591796875, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 116.1591796875 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/moonep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1", "config": "KIMI_K2_L2", "backend": "moonep", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 44.29609251022339, "max_reserved_gib": 44.716796875, "fragmentation_gib": 0.42070436477661133, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 84.4638671875, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 84.4638671875 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/moonep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1", "config": "KIMI_K2_L2", "backend": "moonep", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 44.27890348434448, "max_reserved_gib": 44.71484375, "fragmentation_gib": 0.4359402656555176, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 84.0966796875, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 84.0966796875 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1", "config": "KIMI_K2_L2", "backend": "ultraep", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.17578125", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 47.984477043151855, "max_reserved_gib": 53.119140625, "fragmentation_gib": 5.1346635818481445, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 88.2568359375, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 88.2568359375 }, { "run_dir": "tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "KIMI_K2_L2", "backend": "ultraep", "num_layers": "2", "moe_layers": "1", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 48.07896947860718, "max_reserved_gib": 53.306640625, "fragmentation_gib": 5.227671146392822, "theoretical_weight_optim_gib": 127.651591796875, "peak_sampled_gib": 87.6240234375, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 87.6240234375 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1", "config": "OLMOE_1B_7B_L16", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.08984375", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 22.940572261810303, "max_reserved_gib": 23.759765625, "fragmentation_gib": 0.8191933631896973, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 42.0341796875, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 42.0341796875 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/echo/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "OLMOE_1B_7B_L16", "backend": "echo", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "better-memory-dump-support", "commit": "b7c52442", "status": "OK", "max_allocated_gib": 22.818925857543945, "max_reserved_gib": 23.8125, "fragmentation_gib": 0.9935741424560547, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 37.1337890625, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 37.1337890625 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1", "config": "OLMOE_1B_7B_L16", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 16.60142183303833, "max_reserved_gib": 17.07421875, "fragmentation_gib": 0.4727969169616699, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 35.5673828125, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 35.5673828125 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/moonep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1", "config": "OLMOE_1B_7B_L16", "backend": "moonep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "none", "capacity": "none", "memory_dump_mode": "iteration", "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support", "commit": "7949f384", "status": "OK", "max_allocated_gib": 16.60142183303833, "max_reserved_gib": 17.07421875, "fragmentation_gib": 0.4727969169616699, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 34.9130859375, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 34.9130859375 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__cap1p08984375__memiteration__it6/rep1", "config": "OLMOE_1B_7B_L16", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "1", "capacity_mode": "static", "capacity": "1.08984375", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 21.23912763595581, "max_reserved_gib": 21.6484375, "fragmentation_gib": 0.40930986404418945, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 38.1806640625, "snapshot_pickle_count": 48, "snapshot_json_count": 48, "snapshot_expected_each": 48, "snapshot_complete": 1, "effective_peak_gib": 38.1806640625 }, { "run_dir": "tierb_models/memory/OLMOE_1B_7B_L16/ultraep/tierbmem/tierbmem_gbs8/OLMOE_1B_7B_L16__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", "config": "OLMOE_1B_7B_L16", "backend": "ultraep", "num_layers": "16", "moe_layers": "16", "seq_length": "4096", "global_batch_size": "8", "cuda_graph": "0", "capacity_mode": "dynamic", "capacity": "dynamic", "memory_dump_mode": "iteration", "branch": "ultraep-better-memory-dump-support", "commit": "c3a8a8a7", "status": "OK", "max_allocated_gib": 21.081180095672607, "max_reserved_gib": 21.509765625, "fragmentation_gib": 0.4285855293273926, "theoretical_weight_optim_gib": 48.31515625, "peak_sampled_gib": 36.2216796875, "snapshot_pickle_count": 40, "snapshot_json_count": 40, "snapshot_expected_each": 40, "snapshot_complete": 1, "effective_peak_gib": 36.2216796875 } ], "comparisons": [ { "config": "DEEPSEEK_V3_L4", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 116.5556640625, "cap_1p13671875_status": "OK", "cap_1p13671875_peak_gib": 128.0029296875 }, { "config": "DEEPSEEK_V3_L4", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 95.5556640625, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 81.8642578125, "moon_graph_delta_gib": 13.69140625 }, { "config": "DEEPSEEK_V3_L4", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 100.2666015625, "cap_1p13671875_status": "OK", "cap_1p13671875_peak_gib": 99.6044921875 }, { "config": "G2_K2", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 85.388671875, "cap_1p15625_status": "OK", "cap_1p15625_peak_gib": 88.99609375 }, { "config": "G2_K2", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 56.009765625, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 55.9375, "moon_graph_delta_gib": 0.072265625 }, { "config": "G2_K2", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 78.193359375, "cap_1p15625_status": "OK", "cap_1p15625_peak_gib": 77.595703125 }, { "config": "G3_K2", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 85.119140625, "cap_1p1875_status": "OK", "cap_1p1875_peak_gib": 90.314453125 }, { "config": "G3_K2", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 54.93359375, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 54.931640625, "moon_graph_delta_gib": 0.001953125 }, { "config": "G3_K2", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 78.611328125, "cap_1p1875_status": "OK", "cap_1p1875_peak_gib": 78.802734375 }, { "config": "G3_K8", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 80.931640625, "cap_1p09375_status": "OK", "cap_1p09375_peak_gib": 86.02734375 }, { "config": "G3_K8", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 54.701171875, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 54.62890625, "moon_graph_delta_gib": 0.072265625 }, { "config": "G3_K8", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 72.55859375, "cap_1p09375_status": "OK", "cap_1p09375_peak_gib": 74.333984375 }, { "config": "G4_K16", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 80.279296875, "cap_1p08203125_status": "OK", "cap_1p08203125_peak_gib": 85.353515625 }, { "config": "G4_K16", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 53.533203125, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 53.5078125, "moon_graph_delta_gib": 0.025390625 }, { "config": "G4_K16", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 72.544921875, "cap_1p08203125_status": "OK", "cap_1p08203125_peak_gib": 73.294921875 }, { "config": "G4_K2", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 78.705078125, "cap_1p265625_status": "OK", "cap_1p265625_peak_gib": 83.779296875 }, { "config": "G4_K2", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 54.099609375, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 54.07421875, "moon_graph_delta_gib": 0.025390625 }, { "config": "G4_K2", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 71.435546875, "cap_1p265625_status": "OK", "cap_1p265625_peak_gib": 71.72265625 }, { "config": "G4_K4", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 77.9765625, "cap_1p171875_status": "OK", "cap_1p171875_peak_gib": 82.447265625 }, { "config": "G4_K4", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 53.697265625, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 53.626953125, "moon_graph_delta_gib": 0.0703125 }, { "config": "G4_K4", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 70.23828125, "cap_1p171875_status": "OK", "cap_1p171875_peak_gib": 70.310546875 }, { "config": "G4_K8", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 75.841796875, "cap_1p109375_status": "OK", "cap_1p109375_peak_gib": 80.9453125 }, { "config": "G4_K8", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 53.712890625, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 53.701171875, "moon_graph_delta_gib": 0.01171875 }, { "config": "G4_K8", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 68.630859375, "cap_1p109375_status": "OK", "cap_1p109375_peak_gib": 68.87109375 }, { "config": "KIMI_K2_L2", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 116.1591796875, "cap_1p17578125_status": "OK", "cap_1p17578125_peak_gib": 129.1201171875 }, { "config": "KIMI_K2_L2", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 84.4638671875, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 84.0966796875, "moon_graph_delta_gib": 0.3671875 }, { "config": "KIMI_K2_L2", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 87.6240234375, "cap_1p17578125_status": "OK", "cap_1p17578125_peak_gib": 88.2568359375 }, { "config": "OLMOE_1B_7B_L16", "backend": "echo", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 37.1337890625, "cap_1p08984375_status": "OK", "cap_1p08984375_peak_gib": 42.0341796875 }, { "config": "OLMOE_1B_7B_L16", "backend": "moonep", "moon_graph_on_status": "OK", "moon_graph_on_peak_gib": 35.5673828125, "moon_graph_off_status": "OK", "moon_graph_off_peak_gib": 34.9130859375, "moon_graph_delta_gib": 0.654296875 }, { "config": "OLMOE_1B_7B_L16", "backend": "ultraep", "dynamic_nocg_status": "OK", "dynamic_nocg_peak_gib": 36.2216796875, "cap_1p08984375_status": "OK", "cap_1p08984375_peak_gib": 38.1806640625 } ] }
{
"runs": [
{
"run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__memiteration__it6/rep1",
"config": "DEEPSEEK_V3_L4",
"backend": "echo",
"num_layers": "4",
"moe_layers": "1",
"seq_length": "4096",
"global_batch_size": "8",
"cuda_graph": "1",
"capacity_mode": "static",
"capacity": "1.13671875",
"memory_dump_mode": "iteration",
"branch": "better-memory-dump-support",
"commit": "b7c52442",
"status": "OK",
"max_allocated_gib": 68.58609199523926,
"max_reserved_gib": 82.048828125,
"fragmentation_gib": 13.462736129760742,
"theoretical_weight_optim_gib": 97.629130859375,
"peak_sampled_gib": 128.0029296875,
"snapshot_pickle_count": 48,
"snapshot_json_count": 48,
"snapshot_expected_each": 48,
"snapshot_complete": 1,
"effective_peak_gib": 128.0029296875
},
{
"run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/echo/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1",
"config": "DEEPSEEK_V3_L4",
"backend": "echo",
"num_layers": "4",
"moe_layers": "1",
"seq_length": "4096",
"global_batch_size": "8",
"cuda_graph": "0",
"capacity_mode": "dynamic",
"capacity": "dynamic",
"memory_dump_mode": "iteration",
"branch": "better-memory-dump-support",
"commit": "b7c52442",
"status": "OK",
"max_allocated_gib": 69.96967601776123,
"max_reserved_gib": 79.791015625,
"fragmentation_gib": 9.82133960723877,
"theoretical_weight_optim_gib": 97.629130859375,
"peak_sampled_gib": 116.5556640625,
"snapshot_pickle_count": 40,
"snapshot_json_count": 40,
"snapshot_expected_each": 40,
"snapshot_complete": 1,
"effective_peak_gib": 116.5556640625
},
{
"run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1",
"config": "DEEPSEEK_V3_L4",
"backend": "moonep",
"num_layers": "4",
"moe_layers": "1",
"seq_length": "4096",
"global_batch_size": "8",
"cuda_graph": "1",
"capacity_mode": "none",
"capacity": "none",
"memory_dump_mode": "iteration",
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
"commit": "7949f384",
"status": "OK",
"max_allocated_gib": 56.274383544921875,
"max_reserved_gib": 59.23046875,
"fragmentation_gib": 2.956085205078125,
"theoretical_weight_optim_gib": 97.629130859375,
"peak_sampled_gib": 95.5556640625,
"snapshot_pickle_count": 48,
"snapshot_json_count": 48,
"snapshot_expected_each": 48,
"snapshot_complete": 1,
"effective_peak_gib": 95.5556640625
},
{
"run_dir": "tierb_models/memory/DEEPSEEK_V3_L4/moonep/tierbmem/tierbmem_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1",
"config": "DEEPSEEK_V3_L4",
"backend": "moonep",
"num_layers": "4",
"moe_layers": "1",
"seq_length": "4096",
"global_batch_size": "8",
"cuda_graph": "0",
"capacity_mode": "none",
"capacity": "none",
"memory_dump_mode": "iteration",
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
"commit": "7949f384",
"status": "OK",
"max_allocated_gib": 56.25963544845581,
"max_reserved_gib": 59.21484375,
"fragmentation_gib": 2.9552083015441895,
"theoretical_weight_optim_gib": 97.629130859375,
"peak_sampled_gib": 81.8642578125,