granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:26:09Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)
Browse files
tiera-rerun-fixed-moonep/hf_watcher.log
CHANGED
|
@@ -125,3 +125,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 125 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 126 |
Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
|
| 127 |
[2026-08-30T10:17:11+00:00] backup pass
|
|
|
|
|
|
|
|
|
| 125 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 126 |
Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
|
| 127 |
[2026-08-30T10:17:11+00:00] backup pass
|
| 128 |
+
uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
|
| 129 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
tiera-rerun-fixed-moonep/memory_summary_all.json
CHANGED
|
@@ -467,6 +467,162 @@
|
|
| 467 |
"snapshot_expected_each": 40,
|
| 468 |
"snapshot_complete": 1,
|
| 469 |
"effective_peak_gib": 109.0888671875
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 470 |
}
|
| 471 |
],
|
| 472 |
"comparisons": [
|
|
@@ -544,6 +700,31 @@
|
|
| 544 |
"dynamic_nocg_peak_gib": 109.0888671875,
|
| 545 |
"cap_1p072265625_status": "OK",
|
| 546 |
"cap_1p072265625_peak_gib": 111.2802734375
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 547 |
}
|
| 548 |
]
|
| 549 |
}
|
|
|
|
| 467 |
"snapshot_expected_each": 40,
|
| 468 |
"snapshot_complete": 1,
|
| 469 |
"effective_peak_gib": 109.0888671875
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"run_dir": "echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1",
|
| 473 |
+
"config": "G4",
|
| 474 |
+
"backend": "echo",
|
| 475 |
+
"num_layers": "16",
|
| 476 |
+
"moe_layers": "16",
|
| 477 |
+
"seq_length": "4096",
|
| 478 |
+
"global_batch_size": "16",
|
| 479 |
+
"cuda_graph": "1",
|
| 480 |
+
"capacity_mode": "static",
|
| 481 |
+
"capacity": "1.095703125",
|
| 482 |
+
"memory_dump_mode": "iteration",
|
| 483 |
+
"branch": "better-memory-dump-support",
|
| 484 |
+
"commit": "b7c52442",
|
| 485 |
+
"status": "OK",
|
| 486 |
+
"max_allocated_gib": 92.9096622467041,
|
| 487 |
+
"max_reserved_gib": 95.123046875,
|
| 488 |
+
"fragmentation_gib": 2.2133846282958984,
|
| 489 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 490 |
+
"peak_sampled_gib": 164.3173828125,
|
| 491 |
+
"snapshot_pickle_count": 48,
|
| 492 |
+
"snapshot_json_count": 48,
|
| 493 |
+
"snapshot_expected_each": 48,
|
| 494 |
+
"snapshot_complete": 1,
|
| 495 |
+
"effective_peak_gib": 164.3173828125
|
| 496 |
+
},
|
| 497 |
+
{
|
| 498 |
+
"run_dir": "echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
|
| 499 |
+
"config": "G4",
|
| 500 |
+
"backend": "echo",
|
| 501 |
+
"num_layers": "16",
|
| 502 |
+
"moe_layers": "16",
|
| 503 |
+
"seq_length": "4096",
|
| 504 |
+
"global_batch_size": "16",
|
| 505 |
+
"cuda_graph": "0",
|
| 506 |
+
"capacity_mode": "dynamic",
|
| 507 |
+
"capacity": "dynamic",
|
| 508 |
+
"memory_dump_mode": "iteration",
|
| 509 |
+
"branch": "better-memory-dump-support",
|
| 510 |
+
"commit": "b7c52442",
|
| 511 |
+
"status": "OK",
|
| 512 |
+
"max_allocated_gib": 86.03302049636841,
|
| 513 |
+
"max_reserved_gib": 89.65625,
|
| 514 |
+
"fragmentation_gib": 3.623229503631592,
|
| 515 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 516 |
+
"peak_sampled_gib": 122.2978515625,
|
| 517 |
+
"snapshot_pickle_count": 40,
|
| 518 |
+
"snapshot_json_count": 40,
|
| 519 |
+
"snapshot_expected_each": 40,
|
| 520 |
+
"snapshot_complete": 1,
|
| 521 |
+
"effective_peak_gib": 122.2978515625
|
| 522 |
+
},
|
| 523 |
+
{
|
| 524 |
+
"run_dir": "moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1",
|
| 525 |
+
"config": "G4",
|
| 526 |
+
"backend": "moonep",
|
| 527 |
+
"num_layers": "16",
|
| 528 |
+
"moe_layers": "16",
|
| 529 |
+
"seq_length": "4096",
|
| 530 |
+
"global_batch_size": "16",
|
| 531 |
+
"cuda_graph": "1",
|
| 532 |
+
"capacity_mode": "none",
|
| 533 |
+
"capacity": "none",
|
| 534 |
+
"memory_dump_mode": "iteration",
|
| 535 |
+
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
|
| 536 |
+
"commit": "7949f384",
|
| 537 |
+
"status": "OK",
|
| 538 |
+
"max_allocated_gib": 54.00910758972168,
|
| 539 |
+
"max_reserved_gib": 55.814453125,
|
| 540 |
+
"fragmentation_gib": 1.8053455352783203,
|
| 541 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 542 |
+
"peak_sampled_gib": 103.5439453125,
|
| 543 |
+
"snapshot_pickle_count": 48,
|
| 544 |
+
"snapshot_json_count": 48,
|
| 545 |
+
"snapshot_expected_each": 48,
|
| 546 |
+
"snapshot_complete": 1,
|
| 547 |
+
"effective_peak_gib": 103.5439453125
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"run_dir": "moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1",
|
| 551 |
+
"config": "G4",
|
| 552 |
+
"backend": "moonep",
|
| 553 |
+
"num_layers": "16",
|
| 554 |
+
"moe_layers": "16",
|
| 555 |
+
"seq_length": "4096",
|
| 556 |
+
"global_batch_size": "16",
|
| 557 |
+
"cuda_graph": "0",
|
| 558 |
+
"capacity_mode": "none",
|
| 559 |
+
"capacity": "none",
|
| 560 |
+
"memory_dump_mode": "iteration",
|
| 561 |
+
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
|
| 562 |
+
"commit": "7949f384",
|
| 563 |
+
"status": "OK",
|
| 564 |
+
"max_allocated_gib": 53.99764823913574,
|
| 565 |
+
"max_reserved_gib": 55.783203125,
|
| 566 |
+
"fragmentation_gib": 1.7855548858642578,
|
| 567 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 568 |
+
"peak_sampled_gib": 102.6376953125,
|
| 569 |
+
"snapshot_pickle_count": 40,
|
| 570 |
+
"snapshot_json_count": 40,
|
| 571 |
+
"snapshot_expected_each": 40,
|
| 572 |
+
"snapshot_complete": 1,
|
| 573 |
+
"effective_peak_gib": 102.6376953125
|
| 574 |
+
},
|
| 575 |
+
{
|
| 576 |
+
"run_dir": "ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1",
|
| 577 |
+
"config": "G4",
|
| 578 |
+
"backend": "ultraep",
|
| 579 |
+
"num_layers": "16",
|
| 580 |
+
"moe_layers": "16",
|
| 581 |
+
"seq_length": "4096",
|
| 582 |
+
"global_batch_size": "16",
|
| 583 |
+
"cuda_graph": "1",
|
| 584 |
+
"capacity_mode": "static",
|
| 585 |
+
"capacity": "1.095703125",
|
| 586 |
+
"memory_dump_mode": "iteration",
|
| 587 |
+
"branch": "ultraep-better-memory-dump-support",
|
| 588 |
+
"commit": "c3a8a8a7",
|
| 589 |
+
"status": "OK",
|
| 590 |
+
"max_allocated_gib": 81.24927520751953,
|
| 591 |
+
"max_reserved_gib": 83.044921875,
|
| 592 |
+
"fragmentation_gib": 1.7956466674804688,
|
| 593 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 594 |
+
"peak_sampled_gib": 129.3330078125,
|
| 595 |
+
"snapshot_pickle_count": 48,
|
| 596 |
+
"snapshot_json_count": 48,
|
| 597 |
+
"snapshot_expected_each": 48,
|
| 598 |
+
"snapshot_complete": 1,
|
| 599 |
+
"effective_peak_gib": 129.3330078125
|
| 600 |
+
},
|
| 601 |
+
{
|
| 602 |
+
"run_dir": "ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
|
| 603 |
+
"config": "G4",
|
| 604 |
+
"backend": "ultraep",
|
| 605 |
+
"num_layers": "16",
|
| 606 |
+
"moe_layers": "16",
|
| 607 |
+
"seq_length": "4096",
|
| 608 |
+
"global_batch_size": "16",
|
| 609 |
+
"cuda_graph": "0",
|
| 610 |
+
"capacity_mode": "dynamic",
|
| 611 |
+
"capacity": "dynamic",
|
| 612 |
+
"memory_dump_mode": "iteration",
|
| 613 |
+
"branch": "ultraep-better-memory-dump-support",
|
| 614 |
+
"commit": "c3a8a8a7",
|
| 615 |
+
"status": "OK",
|
| 616 |
+
"max_allocated_gib": 79.67659139633179,
|
| 617 |
+
"max_reserved_gib": 81.693359375,
|
| 618 |
+
"fragmentation_gib": 2.016767978668213,
|
| 619 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 620 |
+
"peak_sampled_gib": 118.1220703125,
|
| 621 |
+
"snapshot_pickle_count": 40,
|
| 622 |
+
"snapshot_json_count": 40,
|
| 623 |
+
"snapshot_expected_each": 40,
|
| 624 |
+
"snapshot_complete": 1,
|
| 625 |
+
"effective_peak_gib": 118.1220703125
|
| 626 |
}
|
| 627 |
],
|
| 628 |
"comparisons": [
|
|
|
|
| 700 |
"dynamic_nocg_peak_gib": 109.0888671875,
|
| 701 |
"cap_1p072265625_status": "OK",
|
| 702 |
"cap_1p072265625_peak_gib": 111.2802734375
|
| 703 |
+
},
|
| 704 |
+
{
|
| 705 |
+
"config": "G4",
|
| 706 |
+
"backend": "echo",
|
| 707 |
+
"dynamic_nocg_status": "OK",
|
| 708 |
+
"dynamic_nocg_peak_gib": 122.2978515625,
|
| 709 |
+
"cap_1p095703125_status": "OK",
|
| 710 |
+
"cap_1p095703125_peak_gib": 164.3173828125
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"config": "G4",
|
| 714 |
+
"backend": "moonep",
|
| 715 |
+
"moon_graph_on_status": "OK",
|
| 716 |
+
"moon_graph_on_peak_gib": 103.5439453125,
|
| 717 |
+
"moon_graph_off_status": "OK",
|
| 718 |
+
"moon_graph_off_peak_gib": 102.6376953125,
|
| 719 |
+
"moon_graph_delta_gib": 0.90625
|
| 720 |
+
},
|
| 721 |
+
{
|
| 722 |
+
"config": "G4",
|
| 723 |
+
"backend": "ultraep",
|
| 724 |
+
"dynamic_nocg_status": "OK",
|
| 725 |
+
"dynamic_nocg_peak_gib": 118.1220703125,
|
| 726 |
+
"cap_1p095703125_status": "OK",
|
| 727 |
+
"cap_1p095703125_peak_gib": 129.3330078125
|
| 728 |
}
|
| 729 |
]
|
| 730 |
}
|
tiera-rerun-fixed-moonep/memory_summary_all.tsv
CHANGED
|
@@ -17,3 +17,9 @@ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__asy
|
|
| 17 |
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 18 |
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 19 |
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 18 |
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 19 |
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 20 |
+
echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1 G4 echo 1 static 1.095703125 OK 164.3173828125 92.9096622467041 95.123046875 2.2133846282958984 164.3173828125 48 48 48 1 better-memory-dump-support b7c52442
|
| 21 |
+
echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G4 echo 0 dynamic dynamic OK 122.2978515625 86.03302049636841 89.65625 3.623229503631592 122.2978515625 40 40 40 1 better-memory-dump-support b7c52442
|
| 22 |
+
moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 G4 moonep 1 none none OK 103.5439453125 54.00910758972168 55.814453125 1.8053455352783203 103.5439453125 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 23 |
+
moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G4 moonep 0 none none OK 102.6376953125 53.99764823913574 55.783203125 1.7855548858642578 102.6376953125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 24 |
+
ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1 G4 ultraep 1 static 1.095703125 OK 129.3330078125 81.24927520751953 83.044921875 1.7956466674804688 129.3330078125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 25 |
+
ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G4 ultraep 0 dynamic dynamic OK 118.1220703125 79.67659139633179 81.693359375 2.016767978668213 118.1220703125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|