AbhishekGhosh1998 commited on
Commit
1894dc2
·
verified ·
1 Parent(s): 069ab73

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:17:03Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)

Browse files
tiera-rerun-fixed-moonep/hf_watcher.log CHANGED
@@ -120,3 +120,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
120
  Already logged in to Hugging Face as: AbhishekGhosh1998
121
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
122
  [2026-08-30T10:08:07+00:00] backup pass
 
 
 
120
  Already logged in to Hugging Face as: AbhishekGhosh1998
121
  Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
122
  [2026-08-30T10:08:07+00:00] backup pass
123
+ uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
124
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
tiera-rerun-fixed-moonep/memory_summary_all.json CHANGED
@@ -311,6 +311,162 @@
311
  "snapshot_expected_each": 40,
312
  "snapshot_complete": 1,
313
  "effective_peak_gib": 105.6767578125
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
314
  }
315
  ],
316
  "comparisons": [
@@ -363,6 +519,31 @@
363
  "dynamic_nocg_peak_gib": 105.6767578125,
364
  "cap_1p08203125_status": "OK",
365
  "cap_1p08203125_peak_gib": 109.6845703125
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
366
  }
367
  ]
368
  }
 
311
  "snapshot_expected_each": 40,
312
  "snapshot_complete": 1,
313
  "effective_peak_gib": 105.6767578125
314
+ },
315
+ {
316
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1",
317
+ "config": "G3",
318
+ "backend": "echo",
319
+ "num_layers": "16",
320
+ "moe_layers": "16",
321
+ "seq_length": "4096",
322
+ "global_batch_size": "16",
323
+ "cuda_graph": "1",
324
+ "capacity_mode": "static",
325
+ "capacity": "1.072265625",
326
+ "memory_dump_mode": "iteration",
327
+ "branch": "better-memory-dump-support",
328
+ "commit": "b7c52442",
329
+ "status": "OK",
330
+ "max_allocated_gib": 86.2518105506897,
331
+ "max_reserved_gib": 89.423828125,
332
+ "fragmentation_gib": 3.1720175743103027,
333
+ "theoretical_weight_optim_gib": 165.067802734375,
334
+ "peak_sampled_gib": 144.5400390625,
335
+ "snapshot_pickle_count": 48,
336
+ "snapshot_json_count": 48,
337
+ "snapshot_expected_each": 48,
338
+ "snapshot_complete": 1,
339
+ "effective_peak_gib": 144.5400390625
340
+ },
341
+ {
342
+ "run_dir": "echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
343
+ "config": "G3",
344
+ "backend": "echo",
345
+ "num_layers": "16",
346
+ "moe_layers": "16",
347
+ "seq_length": "4096",
348
+ "global_batch_size": "16",
349
+ "cuda_graph": "0",
350
+ "capacity_mode": "dynamic",
351
+ "capacity": "dynamic",
352
+ "memory_dump_mode": "iteration",
353
+ "branch": "better-memory-dump-support",
354
+ "commit": "b7c52442",
355
+ "status": "OK",
356
+ "max_allocated_gib": 80.13942766189575,
357
+ "max_reserved_gib": 84.869140625,
358
+ "fragmentation_gib": 4.729712963104248,
359
+ "theoretical_weight_optim_gib": 165.067802734375,
360
+ "peak_sampled_gib": 116.1142578125,
361
+ "snapshot_pickle_count": 40,
362
+ "snapshot_json_count": 40,
363
+ "snapshot_expected_each": 40,
364
+ "snapshot_complete": 1,
365
+ "effective_peak_gib": 116.1142578125
366
+ },
367
+ {
368
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1",
369
+ "config": "G3",
370
+ "backend": "moonep",
371
+ "num_layers": "16",
372
+ "moe_layers": "16",
373
+ "seq_length": "4096",
374
+ "global_batch_size": "16",
375
+ "cuda_graph": "1",
376
+ "capacity_mode": "none",
377
+ "capacity": "none",
378
+ "memory_dump_mode": "iteration",
379
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
380
+ "commit": "7949f384",
381
+ "status": "OK",
382
+ "max_allocated_gib": 51.822237491607666,
383
+ "max_reserved_gib": 54.40234375,
384
+ "fragmentation_gib": 2.580106258392334,
385
+ "theoretical_weight_optim_gib": 165.067802734375,
386
+ "peak_sampled_gib": 97.0810546875,
387
+ "snapshot_pickle_count": 48,
388
+ "snapshot_json_count": 48,
389
+ "snapshot_expected_each": 48,
390
+ "snapshot_complete": 1,
391
+ "effective_peak_gib": 97.0810546875
392
+ },
393
+ {
394
+ "run_dir": "moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1",
395
+ "config": "G3",
396
+ "backend": "moonep",
397
+ "num_layers": "16",
398
+ "moe_layers": "16",
399
+ "seq_length": "4096",
400
+ "global_batch_size": "16",
401
+ "cuda_graph": "0",
402
+ "capacity_mode": "none",
403
+ "capacity": "none",
404
+ "memory_dump_mode": "iteration",
405
+ "branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
406
+ "commit": "7949f384",
407
+ "status": "OK",
408
+ "max_allocated_gib": 51.79174995422363,
409
+ "max_reserved_gib": 54.3203125,
410
+ "fragmentation_gib": 2.528562545776367,
411
+ "theoretical_weight_optim_gib": 165.067802734375,
412
+ "peak_sampled_gib": 96.2998046875,
413
+ "snapshot_pickle_count": 40,
414
+ "snapshot_json_count": 40,
415
+ "snapshot_expected_each": 40,
416
+ "snapshot_complete": 1,
417
+ "effective_peak_gib": 96.2998046875
418
+ },
419
+ {
420
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1",
421
+ "config": "G3",
422
+ "backend": "ultraep",
423
+ "num_layers": "16",
424
+ "moe_layers": "16",
425
+ "seq_length": "4096",
426
+ "global_batch_size": "16",
427
+ "cuda_graph": "1",
428
+ "capacity_mode": "static",
429
+ "capacity": "1.072265625",
430
+ "memory_dump_mode": "iteration",
431
+ "branch": "ultraep-better-memory-dump-support",
432
+ "commit": "c3a8a8a7",
433
+ "status": "OK",
434
+ "max_allocated_gib": 75.15694522857666,
435
+ "max_reserved_gib": 77.732421875,
436
+ "fragmentation_gib": 2.57547664642334,
437
+ "theoretical_weight_optim_gib": 165.067802734375,
438
+ "peak_sampled_gib": 111.2802734375,
439
+ "snapshot_pickle_count": 48,
440
+ "snapshot_json_count": 48,
441
+ "snapshot_expected_each": 48,
442
+ "snapshot_complete": 1,
443
+ "effective_peak_gib": 111.2802734375
444
+ },
445
+ {
446
+ "run_dir": "ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
447
+ "config": "G3",
448
+ "backend": "ultraep",
449
+ "num_layers": "16",
450
+ "moe_layers": "16",
451
+ "seq_length": "4096",
452
+ "global_batch_size": "16",
453
+ "cuda_graph": "0",
454
+ "capacity_mode": "dynamic",
455
+ "capacity": "dynamic",
456
+ "memory_dump_mode": "iteration",
457
+ "branch": "ultraep-better-memory-dump-support",
458
+ "commit": "c3a8a8a7",
459
+ "status": "OK",
460
+ "max_allocated_gib": 74.36631107330322,
461
+ "max_reserved_gib": 77.14453125,
462
+ "fragmentation_gib": 2.7782201766967773,
463
+ "theoretical_weight_optim_gib": 165.067802734375,
464
+ "peak_sampled_gib": 109.0888671875,
465
+ "snapshot_pickle_count": 40,
466
+ "snapshot_json_count": 40,
467
+ "snapshot_expected_each": 40,
468
+ "snapshot_complete": 1,
469
+ "effective_peak_gib": 109.0888671875
470
  }
471
  ],
472
  "comparisons": [
 
519
  "dynamic_nocg_peak_gib": 105.6767578125,
520
  "cap_1p08203125_status": "OK",
521
  "cap_1p08203125_peak_gib": 109.6845703125
522
+ },
523
+ {
524
+ "config": "G3",
525
+ "backend": "echo",
526
+ "dynamic_nocg_status": "OK",
527
+ "dynamic_nocg_peak_gib": 116.1142578125,
528
+ "cap_1p072265625_status": "OK",
529
+ "cap_1p072265625_peak_gib": 144.5400390625
530
+ },
531
+ {
532
+ "config": "G3",
533
+ "backend": "moonep",
534
+ "moon_graph_on_status": "OK",
535
+ "moon_graph_on_peak_gib": 97.0810546875,
536
+ "moon_graph_off_status": "OK",
537
+ "moon_graph_off_peak_gib": 96.2998046875,
538
+ "moon_graph_delta_gib": 0.78125
539
+ },
540
+ {
541
+ "config": "G3",
542
+ "backend": "ultraep",
543
+ "dynamic_nocg_status": "OK",
544
+ "dynamic_nocg_peak_gib": 109.0888671875,
545
+ "cap_1p072265625_status": "OK",
546
+ "cap_1p072265625_peak_gib": 111.2802734375
547
  }
548
  ]
549
  }
tiera-rerun-fixed-moonep/memory_summary_all.tsv CHANGED
@@ -11,3 +11,9 @@ moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__asy
11
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G2 moonep 0 none none OK 93.2470703125 51.80956554412842 54.970703125 3.161137580871582 93.2470703125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
12
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 G2 ultraep 1 static 1.08203125 OK 109.6845703125 72.76167297363281 74.708984375 1.9473114013671875 109.6845703125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
13
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G2 ultraep 0 dynamic dynamic OK 105.6767578125 72.02736711502075 73.939453125 1.912086009979248 105.6767578125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
 
 
 
 
 
 
 
11
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G2 moonep 0 none none OK 93.2470703125 51.80956554412842 54.970703125 3.161137580871582 93.2470703125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
12
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 G2 ultraep 1 static 1.08203125 OK 109.6845703125 72.76167297363281 74.708984375 1.9473114013671875 109.6845703125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
13
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G2 ultraep 0 dynamic dynamic OK 105.6767578125 72.02736711502075 73.939453125 1.912086009979248 105.6767578125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
14
+ echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 echo 1 static 1.072265625 OK 144.5400390625 86.2518105506897 89.423828125 3.1720175743103027 144.5400390625 48 48 48 1 better-memory-dump-support b7c52442
15
+ echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 echo 0 dynamic dynamic OK 116.1142578125 80.13942766189575 84.869140625 4.729712963104248 116.1142578125 40 40 40 1 better-memory-dump-support b7c52442
16
+ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 G3 moonep 1 none none OK 97.0810546875 51.822237491607666 54.40234375 2.580106258392334 97.0810546875 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
17
+ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
18
+ ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
19
+ ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7