AbhishekGhosh1998 commited on
Commit
b53279c
·
verified ·
1 Parent(s): 43eed28

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:26:09Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)

Browse files
tiera-rerun-fixed-moonep/hf_watcher.log CHANGED
@@ -125,3 +125,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
125
  Already logged in to Hugging Face as: AbhishekGhosh1998
126
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
127
  [2026-08-30T10:17:11+00:00] backup pass
 
 
 
125
  Already logged in to Hugging Face as: AbhishekGhosh1998
126
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
127
  [2026-08-30T10:17:11+00:00] backup pass
128
+ uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
129
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
tiera-rerun-fixed-moonep/memory_summary_all.json CHANGED
@@ -467,6 +467,162 @@
467
  "snapshot_expected_each": 40,
468
  "snapshot_complete": 1,
469
  "effective_peak_gib": 109.0888671875
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
470
  }
471
  ],
472
  "comparisons": [
@@ -544,6 +700,31 @@
544
  "dynamic_nocg_peak_gib": 109.0888671875,
545
  "cap_1p072265625_status": "OK",
546
  "cap_1p072265625_peak_gib": 111.2802734375
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
547
  }
548
  ]
549
  }
 
467
  "snapshot_expected_each": 40,
468
  "snapshot_complete": 1,
469
  "effective_peak_gib": 109.0888671875
470
+ },
471
+ {
472
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1",
473
+ "config": "G4",
474
+ "backend": "echo",
475
+ "num_layers": "16",
476
+ "moe_layers": "16",
477
+ "seq_length": "4096",
478
+ "global_batch_size": "16",
479
+ "cuda_graph": "1",
480
+ "capacity_mode": "static",
481
+ "capacity": "1.095703125",
482
+ "memory_dump_mode": "iteration",
483
+ "branch": "better-memory-dump-support",
484
+ "commit": "b7c52442",
485
+ "status": "OK",
486
+ "max_allocated_gib": 92.9096622467041,
487
+ "max_reserved_gib": 95.123046875,
488
+ "fragmentation_gib": 2.2133846282958984,
489
+ "theoretical_weight_optim_gib": 165.067802734375,
490
+ "peak_sampled_gib": 164.3173828125,
491
+ "snapshot_pickle_count": 48,
492
+ "snapshot_json_count": 48,
493
+ "snapshot_expected_each": 48,
494
+ "snapshot_complete": 1,
495
+ "effective_peak_gib": 164.3173828125
496
+ },
497
+ {
498
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
499
+ "config": "G4",
500
+ "backend": "echo",
501
+ "num_layers": "16",
502
+ "moe_layers": "16",
503
+ "seq_length": "4096",
504
+ "global_batch_size": "16",
505
+ "cuda_graph": "0",
506
+ "capacity_mode": "dynamic",
507
+ "capacity": "dynamic",
508
+ "memory_dump_mode": "iteration",
509
+ "branch": "better-memory-dump-support",
510
+ "commit": "b7c52442",
511
+ "status": "OK",
512
+ "max_allocated_gib": 86.03302049636841,
513
+ "max_reserved_gib": 89.65625,
514
+ "fragmentation_gib": 3.623229503631592,
515
+ "theoretical_weight_optim_gib": 165.067802734375,
516
+ "peak_sampled_gib": 122.2978515625,
517
+ "snapshot_pickle_count": 40,
518
+ "snapshot_json_count": 40,
519
+ "snapshot_expected_each": 40,
520
+ "snapshot_complete": 1,
521
+ "effective_peak_gib": 122.2978515625
522
+ },
523
+ {
524
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1",
525
+ "config": "G4",
526
+ "backend": "moonep",
527
+ "num_layers": "16",
528
+ "moe_layers": "16",
529
+ "seq_length": "4096",
530
+ "global_batch_size": "16",
531
+ "cuda_graph": "1",
532
+ "capacity_mode": "none",
533
+ "capacity": "none",
534
+ "memory_dump_mode": "iteration",
535
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
536
+ "commit": "7949f384",
537
+ "status": "OK",
538
+ "max_allocated_gib": 54.00910758972168,
539
+ "max_reserved_gib": 55.814453125,
540
+ "fragmentation_gib": 1.8053455352783203,
541
+ "theoretical_weight_optim_gib": 165.067802734375,
542
+ "peak_sampled_gib": 103.5439453125,
543
+ "snapshot_pickle_count": 48,
544
+ "snapshot_json_count": 48,
545
+ "snapshot_expected_each": 48,
546
+ "snapshot_complete": 1,
547
+ "effective_peak_gib": 103.5439453125
548
+ },
549
+ {
550
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1",
551
+ "config": "G4",
552
+ "backend": "moonep",
553
+ "num_layers": "16",
554
+ "moe_layers": "16",
555
+ "seq_length": "4096",
556
+ "global_batch_size": "16",
557
+ "cuda_graph": "0",
558
+ "capacity_mode": "none",
559
+ "capacity": "none",
560
+ "memory_dump_mode": "iteration",
561
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
562
+ "commit": "7949f384",
563
+ "status": "OK",
564
+ "max_allocated_gib": 53.99764823913574,
565
+ "max_reserved_gib": 55.783203125,
566
+ "fragmentation_gib": 1.7855548858642578,
567
+ "theoretical_weight_optim_gib": 165.067802734375,
568
+ "peak_sampled_gib": 102.6376953125,
569
+ "snapshot_pickle_count": 40,
570
+ "snapshot_json_count": 40,
571
+ "snapshot_expected_each": 40,
572
+ "snapshot_complete": 1,
573
+ "effective_peak_gib": 102.6376953125
574
+ },
575
+ {
576
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1",
577
+ "config": "G4",
578
+ "backend": "ultraep",
579
+ "num_layers": "16",
580
+ "moe_layers": "16",
581
+ "seq_length": "4096",
582
+ "global_batch_size": "16",
583
+ "cuda_graph": "1",
584
+ "capacity_mode": "static",
585
+ "capacity": "1.095703125",
586
+ "memory_dump_mode": "iteration",
587
+ "branch": "ultraep-better-memory-dump-support",
588
+ "commit": "c3a8a8a7",
589
+ "status": "OK",
590
+ "max_allocated_gib": 81.24927520751953,
591
+ "max_reserved_gib": 83.044921875,
592
+ "fragmentation_gib": 1.7956466674804688,
593
+ "theoretical_weight_optim_gib": 165.067802734375,
594
+ "peak_sampled_gib": 129.3330078125,
595
+ "snapshot_pickle_count": 48,
596
+ "snapshot_json_count": 48,
597
+ "snapshot_expected_each": 48,
598
+ "snapshot_complete": 1,
599
+ "effective_peak_gib": 129.3330078125
600
+ },
601
+ {
602
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
603
+ "config": "G4",
604
+ "backend": "ultraep",
605
+ "num_layers": "16",
606
+ "moe_layers": "16",
607
+ "seq_length": "4096",
608
+ "global_batch_size": "16",
609
+ "cuda_graph": "0",
610
+ "capacity_mode": "dynamic",
611
+ "capacity": "dynamic",
612
+ "memory_dump_mode": "iteration",
613
+ "branch": "ultraep-better-memory-dump-support",
614
+ "commit": "c3a8a8a7",
615
+ "status": "OK",
616
+ "max_allocated_gib": 79.67659139633179,
617
+ "max_reserved_gib": 81.693359375,
618
+ "fragmentation_gib": 2.016767978668213,
619
+ "theoretical_weight_optim_gib": 165.067802734375,
620
+ "peak_sampled_gib": 118.1220703125,
621
+ "snapshot_pickle_count": 40,
622
+ "snapshot_json_count": 40,
623
+ "snapshot_expected_each": 40,
624
+ "snapshot_complete": 1,
625
+ "effective_peak_gib": 118.1220703125
626
  }
627
  ],
628
  "comparisons": [
 
700
  "dynamic_nocg_peak_gib": 109.0888671875,
701
  "cap_1p072265625_status": "OK",
702
  "cap_1p072265625_peak_gib": 111.2802734375
703
+ },
704
+ {
705
+ "config": "G4",
706
+ "backend": "echo",
707
+ "dynamic_nocg_status": "OK",
708
+ "dynamic_nocg_peak_gib": 122.2978515625,
709
+ "cap_1p095703125_status": "OK",
710
+ "cap_1p095703125_peak_gib": 164.3173828125
711
+ },
712
+ {
713
+ "config": "G4",
714
+ "backend": "moonep",
715
+ "moon_graph_on_status": "OK",
716
+ "moon_graph_on_peak_gib": 103.5439453125,
717
+ "moon_graph_off_status": "OK",
718
+ "moon_graph_off_peak_gib": 102.6376953125,
719
+ "moon_graph_delta_gib": 0.90625
720
+ },
721
+ {
722
+ "config": "G4",
723
+ "backend": "ultraep",
724
+ "dynamic_nocg_status": "OK",
725
+ "dynamic_nocg_peak_gib": 118.1220703125,
726
+ "cap_1p095703125_status": "OK",
727
+ "cap_1p095703125_peak_gib": 129.3330078125
728
  }
729
  ]
730
  }
tiera-rerun-fixed-moonep/memory_summary_all.tsv CHANGED
@@ -17,3 +17,9 @@ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__asy
17
  moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
18
  ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
19
  ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
 
 
 
 
 
 
 
17
  moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
18
  ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
19
  ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
20
+ echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1 G4 echo 1 static 1.095703125 OK 164.3173828125 92.9096622467041 95.123046875 2.2133846282958984 164.3173828125 48 48 48 1 better-memory-dump-support b7c52442
21
+ echo/tierbmem/tierbmem_gbs16/G4__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G4 echo 0 dynamic dynamic OK 122.2978515625 86.03302049636841 89.65625 3.623229503631592 122.2978515625 40 40 40 1 better-memory-dump-support b7c52442
22
+ moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 G4 moonep 1 none none OK 103.5439453125 54.00910758972168 55.814453125 1.8053455352783203 103.5439453125 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
23
+ moonep/tierbmem/tierbmem_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G4 moonep 0 none none OK 102.6376953125 53.99764823913574 55.783203125 1.7855548858642578 102.6376953125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
24
+ ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125__memiteration__it6/rep1 G4 ultraep 1 static 1.095703125 OK 129.3330078125 81.24927520751953 83.044921875 1.7956466674804688 129.3330078125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
25
+ ultraep/tierbmem/tierbmem_gbs16/G4__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G4 ultraep 0 dynamic dynamic OK 118.1220703125 79.67659139633179 81.693359375 2.016767978668213 118.1220703125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7