granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:17:03Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)
Browse files
tiera-rerun-fixed-moonep/hf_watcher.log
CHANGED
|
@@ -120,3 +120,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 120 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 121 |
Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
|
| 122 |
[2026-08-30T10:08:07+00:00] backup pass
|
|
|
|
|
|
|
|
|
| 120 |
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 121 |
Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop)
|
| 122 |
[2026-08-30T10:08:07+00:00] backup pass
|
| 123 |
+
uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
|
| 124 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
tiera-rerun-fixed-moonep/memory_summary_all.json
CHANGED
|
@@ -311,6 +311,162 @@
|
|
| 311 |
"snapshot_expected_each": 40,
|
| 312 |
"snapshot_complete": 1,
|
| 313 |
"effective_peak_gib": 105.6767578125
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
}
|
| 315 |
],
|
| 316 |
"comparisons": [
|
|
@@ -363,6 +519,31 @@
|
|
| 363 |
"dynamic_nocg_peak_gib": 105.6767578125,
|
| 364 |
"cap_1p08203125_status": "OK",
|
| 365 |
"cap_1p08203125_peak_gib": 109.6845703125
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
}
|
| 367 |
]
|
| 368 |
}
|
|
|
|
| 311 |
"snapshot_expected_each": 40,
|
| 312 |
"snapshot_complete": 1,
|
| 313 |
"effective_peak_gib": 105.6767578125
|
| 314 |
+
},
|
| 315 |
+
{
|
| 316 |
+
"run_dir": "echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1",
|
| 317 |
+
"config": "G3",
|
| 318 |
+
"backend": "echo",
|
| 319 |
+
"num_layers": "16",
|
| 320 |
+
"moe_layers": "16",
|
| 321 |
+
"seq_length": "4096",
|
| 322 |
+
"global_batch_size": "16",
|
| 323 |
+
"cuda_graph": "1",
|
| 324 |
+
"capacity_mode": "static",
|
| 325 |
+
"capacity": "1.072265625",
|
| 326 |
+
"memory_dump_mode": "iteration",
|
| 327 |
+
"branch": "better-memory-dump-support",
|
| 328 |
+
"commit": "b7c52442",
|
| 329 |
+
"status": "OK",
|
| 330 |
+
"max_allocated_gib": 86.2518105506897,
|
| 331 |
+
"max_reserved_gib": 89.423828125,
|
| 332 |
+
"fragmentation_gib": 3.1720175743103027,
|
| 333 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 334 |
+
"peak_sampled_gib": 144.5400390625,
|
| 335 |
+
"snapshot_pickle_count": 48,
|
| 336 |
+
"snapshot_json_count": 48,
|
| 337 |
+
"snapshot_expected_each": 48,
|
| 338 |
+
"snapshot_complete": 1,
|
| 339 |
+
"effective_peak_gib": 144.5400390625
|
| 340 |
+
},
|
| 341 |
+
{
|
| 342 |
+
"run_dir": "echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
|
| 343 |
+
"config": "G3",
|
| 344 |
+
"backend": "echo",
|
| 345 |
+
"num_layers": "16",
|
| 346 |
+
"moe_layers": "16",
|
| 347 |
+
"seq_length": "4096",
|
| 348 |
+
"global_batch_size": "16",
|
| 349 |
+
"cuda_graph": "0",
|
| 350 |
+
"capacity_mode": "dynamic",
|
| 351 |
+
"capacity": "dynamic",
|
| 352 |
+
"memory_dump_mode": "iteration",
|
| 353 |
+
"branch": "better-memory-dump-support",
|
| 354 |
+
"commit": "b7c52442",
|
| 355 |
+
"status": "OK",
|
| 356 |
+
"max_allocated_gib": 80.13942766189575,
|
| 357 |
+
"max_reserved_gib": 84.869140625,
|
| 358 |
+
"fragmentation_gib": 4.729712963104248,
|
| 359 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 360 |
+
"peak_sampled_gib": 116.1142578125,
|
| 361 |
+
"snapshot_pickle_count": 40,
|
| 362 |
+
"snapshot_json_count": 40,
|
| 363 |
+
"snapshot_expected_each": 40,
|
| 364 |
+
"snapshot_complete": 1,
|
| 365 |
+
"effective_peak_gib": 116.1142578125
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"run_dir": "moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1",
|
| 369 |
+
"config": "G3",
|
| 370 |
+
"backend": "moonep",
|
| 371 |
+
"num_layers": "16",
|
| 372 |
+
"moe_layers": "16",
|
| 373 |
+
"seq_length": "4096",
|
| 374 |
+
"global_batch_size": "16",
|
| 375 |
+
"cuda_graph": "1",
|
| 376 |
+
"capacity_mode": "none",
|
| 377 |
+
"capacity": "none",
|
| 378 |
+
"memory_dump_mode": "iteration",
|
| 379 |
+
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
|
| 380 |
+
"commit": "7949f384",
|
| 381 |
+
"status": "OK",
|
| 382 |
+
"max_allocated_gib": 51.822237491607666,
|
| 383 |
+
"max_reserved_gib": 54.40234375,
|
| 384 |
+
"fragmentation_gib": 2.580106258392334,
|
| 385 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 386 |
+
"peak_sampled_gib": 97.0810546875,
|
| 387 |
+
"snapshot_pickle_count": 48,
|
| 388 |
+
"snapshot_json_count": 48,
|
| 389 |
+
"snapshot_expected_each": 48,
|
| 390 |
+
"snapshot_complete": 1,
|
| 391 |
+
"effective_peak_gib": 97.0810546875
|
| 392 |
+
},
|
| 393 |
+
{
|
| 394 |
+
"run_dir": "moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1",
|
| 395 |
+
"config": "G3",
|
| 396 |
+
"backend": "moonep",
|
| 397 |
+
"num_layers": "16",
|
| 398 |
+
"moe_layers": "16",
|
| 399 |
+
"seq_length": "4096",
|
| 400 |
+
"global_batch_size": "16",
|
| 401 |
+
"cuda_graph": "0",
|
| 402 |
+
"capacity_mode": "none",
|
| 403 |
+
"capacity": "none",
|
| 404 |
+
"memory_dump_mode": "iteration",
|
| 405 |
+
"branch": "moonep_backend_async_grad_reduce_better_memory_dump_support",
|
| 406 |
+
"commit": "7949f384",
|
| 407 |
+
"status": "OK",
|
| 408 |
+
"max_allocated_gib": 51.79174995422363,
|
| 409 |
+
"max_reserved_gib": 54.3203125,
|
| 410 |
+
"fragmentation_gib": 2.528562545776367,
|
| 411 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 412 |
+
"peak_sampled_gib": 96.2998046875,
|
| 413 |
+
"snapshot_pickle_count": 40,
|
| 414 |
+
"snapshot_json_count": 40,
|
| 415 |
+
"snapshot_expected_each": 40,
|
| 416 |
+
"snapshot_complete": 1,
|
| 417 |
+
"effective_peak_gib": 96.2998046875
|
| 418 |
+
},
|
| 419 |
+
{
|
| 420 |
+
"run_dir": "ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1",
|
| 421 |
+
"config": "G3",
|
| 422 |
+
"backend": "ultraep",
|
| 423 |
+
"num_layers": "16",
|
| 424 |
+
"moe_layers": "16",
|
| 425 |
+
"seq_length": "4096",
|
| 426 |
+
"global_batch_size": "16",
|
| 427 |
+
"cuda_graph": "1",
|
| 428 |
+
"capacity_mode": "static",
|
| 429 |
+
"capacity": "1.072265625",
|
| 430 |
+
"memory_dump_mode": "iteration",
|
| 431 |
+
"branch": "ultraep-better-memory-dump-support",
|
| 432 |
+
"commit": "c3a8a8a7",
|
| 433 |
+
"status": "OK",
|
| 434 |
+
"max_allocated_gib": 75.15694522857666,
|
| 435 |
+
"max_reserved_gib": 77.732421875,
|
| 436 |
+
"fragmentation_gib": 2.57547664642334,
|
| 437 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 438 |
+
"peak_sampled_gib": 111.2802734375,
|
| 439 |
+
"snapshot_pickle_count": 48,
|
| 440 |
+
"snapshot_json_count": 48,
|
| 441 |
+
"snapshot_expected_each": 48,
|
| 442 |
+
"snapshot_complete": 1,
|
| 443 |
+
"effective_peak_gib": 111.2802734375
|
| 444 |
+
},
|
| 445 |
+
{
|
| 446 |
+
"run_dir": "ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1",
|
| 447 |
+
"config": "G3",
|
| 448 |
+
"backend": "ultraep",
|
| 449 |
+
"num_layers": "16",
|
| 450 |
+
"moe_layers": "16",
|
| 451 |
+
"seq_length": "4096",
|
| 452 |
+
"global_batch_size": "16",
|
| 453 |
+
"cuda_graph": "0",
|
| 454 |
+
"capacity_mode": "dynamic",
|
| 455 |
+
"capacity": "dynamic",
|
| 456 |
+
"memory_dump_mode": "iteration",
|
| 457 |
+
"branch": "ultraep-better-memory-dump-support",
|
| 458 |
+
"commit": "c3a8a8a7",
|
| 459 |
+
"status": "OK",
|
| 460 |
+
"max_allocated_gib": 74.36631107330322,
|
| 461 |
+
"max_reserved_gib": 77.14453125,
|
| 462 |
+
"fragmentation_gib": 2.7782201766967773,
|
| 463 |
+
"theoretical_weight_optim_gib": 165.067802734375,
|
| 464 |
+
"peak_sampled_gib": 109.0888671875,
|
| 465 |
+
"snapshot_pickle_count": 40,
|
| 466 |
+
"snapshot_json_count": 40,
|
| 467 |
+
"snapshot_expected_each": 40,
|
| 468 |
+
"snapshot_complete": 1,
|
| 469 |
+
"effective_peak_gib": 109.0888671875
|
| 470 |
}
|
| 471 |
],
|
| 472 |
"comparisons": [
|
|
|
|
| 519 |
"dynamic_nocg_peak_gib": 105.6767578125,
|
| 520 |
"cap_1p08203125_status": "OK",
|
| 521 |
"cap_1p08203125_peak_gib": 109.6845703125
|
| 522 |
+
},
|
| 523 |
+
{
|
| 524 |
+
"config": "G3",
|
| 525 |
+
"backend": "echo",
|
| 526 |
+
"dynamic_nocg_status": "OK",
|
| 527 |
+
"dynamic_nocg_peak_gib": 116.1142578125,
|
| 528 |
+
"cap_1p072265625_status": "OK",
|
| 529 |
+
"cap_1p072265625_peak_gib": 144.5400390625
|
| 530 |
+
},
|
| 531 |
+
{
|
| 532 |
+
"config": "G3",
|
| 533 |
+
"backend": "moonep",
|
| 534 |
+
"moon_graph_on_status": "OK",
|
| 535 |
+
"moon_graph_on_peak_gib": 97.0810546875,
|
| 536 |
+
"moon_graph_off_status": "OK",
|
| 537 |
+
"moon_graph_off_peak_gib": 96.2998046875,
|
| 538 |
+
"moon_graph_delta_gib": 0.78125
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"config": "G3",
|
| 542 |
+
"backend": "ultraep",
|
| 543 |
+
"dynamic_nocg_status": "OK",
|
| 544 |
+
"dynamic_nocg_peak_gib": 109.0888671875,
|
| 545 |
+
"cap_1p072265625_status": "OK",
|
| 546 |
+
"cap_1p072265625_peak_gib": 111.2802734375
|
| 547 |
}
|
| 548 |
]
|
| 549 |
}
|
tiera-rerun-fixed-moonep/memory_summary_all.tsv
CHANGED
|
@@ -11,3 +11,9 @@ moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__asy
|
|
| 11 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G2 moonep 0 none none OK 93.2470703125 51.80956554412842 54.970703125 3.161137580871582 93.2470703125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 12 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 G2 ultraep 1 static 1.08203125 OK 109.6845703125 72.76167297363281 74.708984375 1.9473114013671875 109.6845703125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 13 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G2 ultraep 0 dynamic dynamic OK 105.6767578125 72.02736711502075 73.939453125 1.912086009979248 105.6767578125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G2 moonep 0 none none OK 93.2470703125 51.80956554412842 54.970703125 3.161137580871582 93.2470703125 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 12 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 G2 ultraep 1 static 1.08203125 OK 109.6845703125 72.76167297363281 74.708984375 1.9473114013671875 109.6845703125 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 13 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G2 ultraep 0 dynamic dynamic OK 105.6767578125 72.02736711502075 73.939453125 1.912086009979248 105.6767578125 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 14 |
+
echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 echo 1 static 1.072265625 OK 144.5400390625 86.2518105506897 89.423828125 3.1720175743103027 144.5400390625 48 48 48 1 better-memory-dump-support b7c52442
|
| 15 |
+
echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 echo 0 dynamic dynamic OK 116.1142578125 80.13942766189575 84.869140625 4.729712963104248 116.1142578125 40 40 40 1 better-memory-dump-support b7c52442
|
| 16 |
+
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 G3 moonep 1 none none OK 97.0810546875 51.822237491607666 54.40234375 2.580106258392334 97.0810546875 48 48 48 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 17 |
+
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 G3 moonep 0 none none OK 96.2998046875 51.79174995422363 54.3203125 2.528562545776367 96.2998046875 40 40 40 1 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384
|
| 18 |
+
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 G3 ultraep 1 static 1.072265625 OK 111.2802734375 75.15694522857666 77.732421875 2.57547664642334 111.2802734375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7
|
| 19 |
+
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 G3 ultraep 0 dynamic dynamic OK 109.0888671875 74.36631107330322 77.14453125 2.7782201766967773 109.0888671875 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7
|