diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv index b40498ef3f5df00db1fe7a10798c504c99549c3a..5023bbcfbf1d95a57d4553e274cdd5adc036521a 100644 --- a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -174,3 +174,51 @@ 5, 68079 6, 68077 7, 68083 +0, 77077 +1, 77103 +2, 77077 +3, 77089 +4, 77145 +5, 77251 +6, 77109 +7, 77129 +0, 86099 +1, 86101 +2, 86101 +3, 86101 +4, 86101 +5, 86157 +6, 86101 +7, 86101 +0, 84189 +1, 84191 +2, 83929 +3, 84191 +4, 84191 +5, 83859 +6, 83971 +7, 84191 +0, 82243 +1, 82769 +2, 82449 +3, 82391 +4, 82769 +5, 82499 +6, 81925 +7, 82769 +0, 388 +1, 68993 +2, 388 +3, 68993 +4, 74369 +5, 324 +6, 68 +7, 68993 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log index aeb3656da3a0b68c0a4201a1352fbb39d67c3622..fc818064026a2c8f0539c8b03168bdc46002a39f 100644 --- a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log @@ -858,3 +858,401 @@ INFO:moonep.api:MoonEP comm buffer sizes: total_size : 619.29 MiB [MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431] sizes [768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[317, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [214, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [92, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [155, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [134, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [219, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-09-21 07:42:54] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25229.1 | throughput per GPU (TFLOP/s/GPU): 1.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.228655E+01 | loss scale: 1.0 | grad norm: 12.484 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 17.55 + +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 45340.978515625 | max allocated: 45340.97900390625 | reserved: 45788.0 | max reserved: 45788.0 +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/distributed/param_and_grad_buffer.py:368: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431] sizes [768, 768, 768, 768, 768, 768, 768, 768, 384, 768, 640, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[201, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [119, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [206, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [8, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [217, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [132, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [274, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] + [2026-09-21 07:42:55] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 139.4 | throughput per GPU (TFLOP/s/GPU): 253.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227169E+01 | loss scale: 1.0 | grad norm: 12.417 | number of skipped iterations: 0 | number of nan iterations: 0 | +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431] sizes [768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 768, 768, 768, 640, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[145, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [76, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [82, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [182, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [101, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [144, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [71, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43152 payload_rows_remote=37792 dup_rows=22384 dup_rows_remote=19584 dedup_ratio=0.341553 router_tokens_per_expert=[92, 87, 82, 76, 100, 82, 107, 85, 73, 91, 78, 89, 75, 87, 77, 71, 76, 66, 67, 73, 80, 88, 93, 79, 95, 101, 81, 98, 75, 91, 95, 91, 82, 71, 91, 77, 78, 83, 91, 97, 87, 75, 61, 87, 80, 89, 95, 83, 78, 100, 85, 65, 90, 83, 80, 96, 81, 88, 70, 80, 87, 98, 84, 72, 75, 81, 98, 115, 69, 85, 86, 107, 101, 96, 77, 92, 96, 79, 87, 83, 80, 87, 98, 74, 79, 107, 82, 86, 95, 95, 77, 80, 71, 98, 93, 79, 89, 84, 93, 78, 83, 94, 85, 86, 85, 88, 79, 91, 64, 79, 82, 95, 103, 88, 78, 72, 97, 92, 99, 77, 99, 77, 79, 84, 88, 70, 79, 94, 69, 75, 80, 80, 107, 76, 69, 73, 84, 77, 94, 79, 92, 79, 89, 85, 89, 87, 81, 81, 94, 85, 89, 87, 88, 107, 89, 88, 97, 88, 85, 101, 78, 105, 71, 80, 86, 85, 91, 85, 74, 70, 88, 69, 82, 99, 91, 88, 73, 82, 67, 78, 86, 90, 98, 79, 82, 83, 80, 101, 81, 72, 97, 101, 88, 73, 77, 83, 91, 82, 84, 74, 83, 76, 86, 81, 89, 115, 89, 93, 93, 86, 104, 82, 80, 70, 86, 72, 78, 73, 96, 98, 89, 83, 99, 98, 100, 74, 82, 76, 71, 92, 83, 76, 74, 86, 102, 79, 86, 77, 93, 78, 94, 103, 86, 82, 86, 78, 93, 95, 82, 90, 80, 78, 87, 82, 83, 90, 77, 76, 90, 79, 86, 90, 91, 93, 84, 80, 82, 83, 93, 90, 79, 83, 94, 82, 79, 90, 83, 94, 87, 87, 84, 91, 80, 88, 89, 79, 87, 79, 89, 84, 73, 68, 73, 90, 92, 92, 87, 101, 95, 87, 108, 97, 86, 80, 86, 83, 73, 92, 89, 66, 84, 96, 91, 99, 87, 99, 94, 93, 84, 83, 83, 83, 78, 78, 84, 97, 83, 101, 78, 73, 77, 79, 75, 76, 97, 91, 74, 101, 98, 77, 86, 78, 87, 84, 101, 98, 83, 77, 95, 89, 78, 90, 92, 88, 86, 84, 102, 79, 77, 84, 98, 85, 85, 91, 77, 76, 71, 82, 75, 105, 94, 92, 71, 68, 81, 95, 68, 81, 83, 74, 81, 76, 107, 92] payload_rows_per_dst=[5364, 5472, 5330, 5430, 5360, 5398, 5428, 5370] dup_rows_per_dst=[2792, 2818, 2748, 2726, 2800, 2838, 2840, 2822] combine_rows_per_src=[5364, 5472, 5330, 5430, 5360, 5398, 5428, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43144 payload_rows_remote=37800 dup_rows=22392 dup_rows_remote=19552 dedup_ratio=0.341675 router_tokens_per_expert=[70, 80, 81, 81, 71, 91, 76, 91, 69, 80, 75, 96, 85, 83, 98, 94, 72, 85, 79, 75, 71, 84, 75, 98, 84, 73, 94, 88, 89, 84, 79, 76, 95, 70, 88, 93, 76, 78, 74, 80, 76, 75, 92, 84, 91, 105, 95, 97, 90, 81, 100, 88, 102, 98, 84, 78, 85, 81, 82, 74, 86, 81, 101, 97, 78, 75, 100, 71, 79, 83, 96, 103, 73, 72, 84, 90, 92, 97, 77, 104, 83, 78, 85, 75, 94, 94, 74, 87, 77, 76, 90, 86, 89, 85, 89, 86, 92, 91, 76, 97, 87, 84, 86, 79, 93, 88, 80, 94, 105, 83, 111, 83, 76, 94, 87, 81, 77, 88, 70, 96, 81, 99, 63, 96, 71, 108, 87, 89, 94, 92, 62, 84, 90, 70, 98, 82, 91, 95, 88, 71, 83, 79, 87, 82, 99, 96, 80, 99, 101, 100, 90, 85, 97, 90, 101, 92, 77, 95, 78, 79, 91, 80, 71, 100, 89, 87, 76, 83, 85, 71, 66, 93, 92, 95, 81, 113, 94, 108, 99, 79, 96, 82, 111, 79, 72, 89, 82, 91, 72, 101, 81, 81, 89, 83, 75, 84, 74, 82, 96, 71, 80, 82, 94, 96, 76, 97, 79, 87, 91, 88, 91, 88, 85, 77, 89, 74, 94, 84, 96, 79, 86, 84, 85, 92, 75, 102, 93, 98, 82, 90, 88, 81, 85, 83, 85, 106, 78, 82, 98, 78, 68, 76, 107, 84, 86, 91, 87, 85, 97, 77, 75, 68, 94, 87, 80, 99, 87, 88, 71, 89, 71, 72, 90, 94, 80, 91, 87, 100, 78, 84, 93, 91, 88, 84, 84, 78, 82, 108, 75, 107, 96, 104, 79, 79, 95, 74, 81, 69, 93, 79, 81, 93, 89, 80, 81, 69, 81, 72, 70, 90, 85, 78, 87, 94, 82, 83, 72, 91, 91, 86, 85, 75, 93, 84, 75, 97, 67, 69, 87, 75, 81, 81, 89, 79, 76, 87, 89, 94, 73, 82, 83, 93, 78, 104, 93, 87, 90, 81, 87, 111, 79, 64, 81, 81, 82, 86, 89, 80, 79, 74, 77, 68, 86, 97, 84, 91, 93, 80, 84, 92, 97, 80, 75, 80, 89, 74, 92, 76, 91, 77, 81, 91, 84, 79, 76, 76, 81, 90, 95, 88, 87, 79, 62, 92] payload_rows_per_dst=[5344, 5440, 5436, 5394, 5372, 5432, 5358, 5368] dup_rows_per_dst=[2840, 2820, 2844, 2912, 2892, 2788, 2648, 2648] combine_rows_per_src=[5344, 5440, 5436, 5394, 5372, 5432, 5358, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43438 payload_rows_remote=37918 dup_rows=22098 dup_rows_remote=19430 dedup_ratio=0.337189 router_tokens_per_expert=[82, 77, 90, 100, 85, 94, 92, 76, 88, 72, 68, 96, 81, 84, 81, 89, 88, 75, 71, 84, 81, 84, 83, 88, 86, 94, 82, 77, 78, 76, 69, 91, 87, 78, 72, 81, 83, 97, 82, 87, 106, 91, 71, 79, 76, 87, 86, 83, 95, 92, 78, 85, 87, 73, 91, 85, 96, 81, 72, 90, 79, 95, 72, 86, 76, 95, 97, 84, 90, 79, 107, 79, 93, 80, 77, 93, 91, 87, 75, 96, 91, 85, 89, 92, 81, 86, 94, 96, 89, 88, 97, 89, 99, 89, 64, 78, 87, 91, 78, 95, 82, 83, 91, 81, 75, 90, 83, 83, 82, 92, 70, 82, 82, 80, 94, 88, 84, 96, 75, 80, 85, 100, 98, 93, 90, 106, 68, 86, 76, 90, 86, 79, 95, 88, 95, 97, 79, 92, 93, 89, 83, 83, 87, 102, 95, 116, 88, 90, 90, 86, 92, 89, 89, 79, 102, 81, 87, 85, 102, 79, 78, 90, 72, 68, 77, 90, 102, 92, 66, 86, 82, 110, 99, 90, 73, 95, 85, 84, 98, 82, 80, 81, 89, 79, 83, 78, 82, 82, 87, 94, 81, 84, 83, 91, 73, 74, 63, 82, 82, 95, 73, 82, 78, 86, 83, 74, 85, 89, 75, 84, 85, 77, 99, 75, 80, 77, 95, 81, 101, 80, 85, 88, 81, 89, 87, 70, 88, 92, 76, 86, 90, 71, 84, 90, 88, 84, 78, 88, 103, 101, 94, 81, 82, 94, 95, 82, 81, 89, 79, 82, 82, 91, 69, 77, 78, 69, 90, 100, 95, 91, 83, 96, 78, 93, 82, 79, 86, 87, 100, 80, 72, 93, 85, 83, 83, 74, 88, 106, 73, 77, 84, 89, 99, 77, 60, 97, 79, 82, 102, 82, 86, 95, 86, 63, 81, 92, 97, 84, 91, 86, 89, 96, 102, 80, 72, 101, 104, 96, 68, 90, 90, 75, 78, 87, 76, 76, 91, 76, 81, 82, 93, 93, 87, 95, 86, 96, 92, 91, 83, 71, 89, 84, 95, 84, 92, 76, 77, 91, 78, 91, 62, 71, 89, 101, 84, 88, 98, 72, 86, 82, 78, 89, 93, 67, 77, 96, 89, 87, 67, 77, 82, 59, 80, 89, 74, 93, 94, 79, 87, 90, 65, 83, 83, 77, 112, 79, 87, 74, 87, 99, 86, 82, 85, 99] payload_rows_per_dst=[5328, 5420, 5464, 5342, 5388, 5438, 5538, 5520] dup_rows_per_dst=[2688, 2698, 2792, 2672, 2832, 2806, 2942, 2668] combine_rows_per_src=[5328, 5420, 5464, 5342, 5388, 5438, 5538, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43366 payload_rows_remote=37964 dup_rows=22170 dup_rows_remote=19470 dedup_ratio=0.338287 router_tokens_per_expert=[92, 79, 88, 92, 80, 77, 71, 85, 109, 103, 96, 98, 83, 86, 83, 85, 79, 90, 78, 80, 98, 85, 79, 71, 94, 77, 87, 88, 78, 87, 98, 86, 78, 89, 72, 88, 82, 84, 83, 78, 88, 81, 84, 69, 92, 95, 73, 59, 75, 78, 86, 90, 85, 70, 88, 74, 72, 102, 73, 90, 81, 70, 81, 103, 95, 81, 87, 93, 98, 85, 66, 91, 101, 68, 73, 100, 101, 84, 89, 84, 105, 102, 99, 81, 73, 74, 67, 80, 88, 81, 78, 78, 79, 90, 72, 90, 73, 96, 80, 74, 86, 91, 64, 94, 83, 94, 104, 82, 74, 84, 72, 82, 81, 97, 77, 80, 91, 78, 80, 81, 83, 81, 102, 92, 79, 93, 67, 69, 80, 91, 79, 91, 92, 83, 97, 97, 78, 80, 96, 87, 92, 80, 95, 87, 87, 96, 87, 81, 102, 105, 87, 99, 91, 93, 92, 93, 88, 84, 93, 79, 100, 75, 81, 81, 86, 82, 79, 99, 81, 76, 104, 86, 83, 99, 79, 66, 83, 83, 97, 79, 84, 95, 98, 95, 86, 80, 86, 84, 103, 79, 101, 80, 73, 86, 74, 81, 110, 75, 103, 94, 83, 74, 72, 80, 68, 105, 73, 84, 97, 104, 74, 83, 67, 80, 87, 90, 87, 92, 92, 80, 88, 88, 75, 91, 86, 76, 78, 85, 77, 95, 73, 94, 91, 91, 87, 79, 79, 75, 76, 89, 95, 101, 97, 94, 87, 96, 93, 83, 89, 77, 69, 63, 94, 90, 92, 85, 90, 87, 90, 88, 84, 75, 89, 90, 88, 78, 83, 67, 86, 84, 75, 81, 83, 76, 89, 73, 85, 95, 74, 87, 82, 88, 79, 92, 79, 78, 87, 92, 81, 99, 91, 82, 99, 78, 85, 77, 88, 77, 93, 73, 83, 92, 91, 103, 74, 87, 97, 70, 80, 78, 79, 74, 59, 92, 94, 87, 112, 88, 83, 73, 74, 78, 94, 82, 111, 90, 82, 74, 75, 72, 73, 87, 86, 104, 87, 76, 75, 61, 83, 89, 93, 89, 107, 91, 109, 92, 89, 88, 83, 85, 79, 79, 84, 89, 98, 92, 73, 93, 99, 101, 88, 93, 91, 106, 83, 80, 80, 88, 81, 99, 75, 82, 77, 96, 86, 96, 86, 91, 80, 76, 67, 80, 96, 82] payload_rows_per_dst=[5462, 5402, 5370, 5456, 5346, 5446, 5384, 5500] dup_rows_per_dst=[2844, 2700, 2768, 2806, 2736, 2712, 2744, 2860] combine_rows_per_src=[5462, 5402, 5370, 5456, 5346, 5446, 5384, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43228 payload_rows_remote=37860 dup_rows=22308 dup_rows_remote=19470 dedup_ratio=0.340393 router_tokens_per_expert=[75, 84, 85, 87, 85, 92, 109, 88, 86, 85, 80, 80, 80, 91, 84, 85, 76, 91, 77, 80, 83, 77, 84, 82, 91, 84, 85, 86, 77, 81, 76, 79, 85, 79, 78, 82, 86, 74, 79, 91, 76, 90, 94, 95, 91, 86, 90, 61, 89, 102, 108, 93, 86, 67, 73, 78, 80, 83, 90, 81, 97, 86, 79, 74, 84, 91, 93, 98, 87, 101, 96, 96, 89, 78, 94, 73, 84, 100, 69, 93, 83, 89, 77, 77, 96, 89, 94, 90, 94, 74, 91, 87, 93, 89, 82, 83, 87, 79, 86, 95, 80, 100, 116, 95, 79, 98, 89, 92, 71, 74, 77, 81, 76, 105, 70, 81, 83, 76, 80, 77, 70, 88, 84, 68, 72, 79, 80, 61, 87, 105, 81, 79, 71, 77, 93, 93, 93, 77, 77, 81, 83, 81, 94, 78, 106, 97, 77, 89, 81, 91, 92, 75, 97, 82, 103, 104, 70, 96, 96, 78, 91, 92, 81, 87, 91, 81, 78, 87, 85, 88, 78, 75, 94, 101, 87, 73, 84, 90, 93, 96, 87, 83, 96, 85, 71, 67, 98, 95, 90, 93, 78, 91, 88, 97, 94, 75, 85, 77, 75, 82, 69, 78, 83, 86, 89, 81, 83, 79, 86, 69, 86, 74, 86, 91, 91, 93, 95, 94, 85, 74, 92, 83, 79, 90, 85, 89, 88, 101, 77, 89, 109, 83, 82, 76, 79, 80, 86, 65, 121, 91, 91, 95, 73, 80, 85, 96, 89, 92, 97, 79, 83, 85, 81, 74, 96, 95, 75, 87, 87, 91, 83, 92, 86, 101, 86, 102, 76, 72, 83, 86, 102, 92, 89, 74, 97, 93, 74, 92, 92, 84, 87, 103, 97, 76, 91, 81, 82, 93, 90, 91, 79, 84, 93, 84, 89, 88, 87, 81, 99, 88, 89, 89, 95, 87, 102, 100, 67, 97, 82, 79, 79, 83, 90, 91, 84, 84, 82, 81, 85, 78, 65, 78, 68, 88, 86, 73, 83, 76, 75, 90, 90, 79, 98, 74, 71, 95, 81, 88, 91, 85, 91, 89, 69, 94, 96, 94, 68, 81, 93, 84, 74, 70, 71, 64, 81, 80, 88, 82, 94, 89, 87, 65, 85, 91, 86, 81, 96, 93, 77, 82, 95, 79, 96, 70, 98, 70, 91, 97, 90, 64, 73, 89, 71, 91] payload_rows_per_dst=[5470, 5474, 5286, 5368, 5470, 5472, 5366, 5322] dup_rows_per_dst=[2768, 2886, 2712, 2838, 2710, 2922, 2766, 2706] combine_rows_per_src=[5470, 5474, 5286, 5368, 5470, 5472, 5366, 5322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=42972 payload_rows_remote=37594 dup_rows=22564 dup_rows_remote=19752 dedup_ratio=0.344299 router_tokens_per_expert=[77, 85, 85, 90, 83, 76, 83, 81, 88, 99, 86, 88, 77, 85, 72, 94, 91, 76, 80, 88, 85, 87, 96, 88, 87, 82, 76, 108, 94, 75, 80, 69, 79, 74, 93, 94, 71, 71, 97, 92, 70, 77, 95, 69, 87, 81, 77, 81, 75, 86, 76, 68, 83, 97, 88, 97, 80, 71, 89, 81, 76, 78, 73, 83, 83, 92, 83, 81, 71, 88, 84, 98, 100, 92, 87, 89, 101, 94, 72, 85, 74, 77, 96, 68, 92, 81, 90, 82, 80, 74, 89, 80, 85, 81, 80, 85, 68, 76, 98, 93, 68, 102, 98, 101, 94, 86, 94, 66, 104, 81, 88, 74, 71, 88, 98, 87, 87, 93, 108, 96, 78, 90, 68, 91, 86, 101, 93, 81, 83, 96, 88, 90, 89, 86, 86, 88, 86, 96, 81, 74, 94, 88, 76, 84, 95, 101, 95, 83, 92, 87, 88, 82, 83, 85, 75, 96, 86, 85, 98, 84, 85, 78, 81, 113, 85, 94, 93, 79, 61, 78, 99, 70, 89, 93, 82, 86, 86, 92, 88, 88, 92, 89, 90, 86, 98, 94, 72, 94, 91, 85, 95, 82, 93, 76, 94, 87, 86, 82, 93, 76, 79, 84, 97, 76, 84, 100, 89, 93, 77, 73, 72, 83, 89, 75, 98, 81, 70, 91, 89, 74, 93, 83, 82, 95, 66, 72, 91, 88, 90, 96, 89, 85, 94, 99, 87, 78, 96, 86, 84, 83, 99, 77, 82, 82, 74, 86, 87, 93, 74, 87, 76, 68, 83, 90, 94, 83, 77, 83, 84, 89, 80, 79, 83, 82, 104, 67, 95, 72, 87, 83, 79, 86, 87, 88, 88, 86, 86, 95, 87, 96, 92, 92, 77, 64, 84, 84, 74, 78, 77, 97, 91, 79, 90, 83, 80, 93, 93, 91, 84, 87, 80, 87, 75, 87, 85, 91, 115, 69, 83, 103, 75, 99, 84, 87, 83, 118, 86, 98, 79, 74, 79, 95, 77, 90, 78, 73, 82, 90, 64, 78, 75, 81, 72, 81, 91, 86, 78, 88, 92, 101, 67, 92, 78, 79, 86, 94, 85, 85, 83, 85, 69, 84, 76, 86, 86, 85, 81, 95, 83, 95, 88, 71, 93, 77, 81, 92, 82, 93, 88, 103, 76, 89, 94, 74, 102, 78, 90, 100, 73, 100, 95, 89, 69, 93] payload_rows_per_dst=[5242, 5280, 5482, 5422, 5422, 5326, 5378, 5420] dup_rows_per_dst=[2796, 2662, 2902, 2848, 2910, 2720, 2812, 2914] combine_rows_per_src=[5242, 5280, 5482, 5422, 5422, 5326, 5378, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43246 payload_rows_remote=37912 dup_rows=22290 dup_rows_remote=19574 dedup_ratio=0.340118 router_tokens_per_expert=[81, 81, 81, 96, 77, 89, 101, 95, 91, 95, 105, 70, 94, 97, 92, 109, 80, 65, 69, 100, 83, 86, 87, 91, 77, 88, 82, 82, 90, 90, 85, 88, 83, 94, 96, 78, 87, 71, 82, 75, 89, 89, 77, 88, 84, 92, 76, 69, 80, 101, 74, 85, 80, 80, 88, 92, 94, 89, 85, 87, 98, 78, 84, 82, 82, 92, 74, 97, 88, 100, 87, 79, 91, 94, 74, 80, 97, 98, 63, 84, 83, 93, 102, 85, 73, 80, 90, 105, 93, 75, 79, 76, 86, 76, 86, 79, 95, 92, 78, 93, 84, 86, 95, 89, 82, 90, 101, 89, 76, 82, 80, 91, 77, 80, 88, 76, 82, 95, 79, 70, 92, 90, 86, 87, 66, 89, 79, 73, 72, 74, 96, 86, 85, 104, 87, 92, 92, 80, 113, 81, 83, 90, 77, 83, 89, 97, 77, 94, 88, 99, 88, 85, 107, 81, 86, 83, 84, 88, 95, 97, 83, 83, 68, 94, 69, 71, 85, 80, 86, 76, 83, 88, 91, 85, 92, 82, 87, 83, 88, 75, 84, 63, 95, 96, 88, 88, 83, 92, 84, 96, 100, 83, 88, 88, 83, 79, 72, 84, 96, 66, 74, 95, 100, 108, 85, 96, 76, 79, 80, 86, 77, 96, 100, 79, 91, 65, 93, 85, 86, 79, 77, 97, 80, 80, 87, 83, 94, 83, 77, 80, 88, 78, 81, 67, 79, 88, 78, 68, 97, 87, 77, 83, 78, 90, 76, 92, 72, 72, 90, 75, 112, 84, 90, 87, 88, 73, 92, 74, 87, 97, 92, 67, 85, 80, 80, 99, 89, 79, 76, 83, 80, 80, 101, 79, 83, 79, 77, 80, 87, 85, 99, 70, 100, 85, 87, 86, 70, 78, 91, 77, 89, 92, 81, 96, 69, 78, 100, 87, 81, 93, 95, 82, 88, 77, 77, 81, 90, 87, 83, 86, 90, 84, 91, 94, 84, 88, 84, 87, 88, 77, 99, 89, 81, 92, 95, 96, 70, 110, 81, 78, 85, 71, 73, 85, 88, 92, 86, 76, 100, 106, 86, 74, 106, 83, 77, 84, 79, 72, 92, 80, 80, 92, 73, 85, 65, 83, 94, 84, 86, 96, 80, 88, 91, 66, 84, 82, 90, 89, 92, 88, 85, 72, 74, 104, 93, 84, 95, 84, 73, 77, 96, 80, 89, 90] payload_rows_per_dst=[5456, 5426, 5434, 5398, 5384, 5334, 5426, 5388] dup_rows_per_dst=[2798, 2810, 2780, 2880, 2686, 2716, 2838, 2782] combine_rows_per_src=[5456, 5426, 5434, 5398, 5384, 5334, 5426, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43500 payload_rows_remote=37974 dup_rows=22036 dup_rows_remote=19374 dedup_ratio=0.336243 router_tokens_per_expert=[83, 92, 93, 80, 85, 84, 84, 76, 94, 96, 74, 96, 68, 97, 74, 90, 81, 93, 83, 95, 95, 79, 89, 87, 84, 84, 94, 85, 70, 87, 66, 81, 85, 70, 91, 86, 93, 81, 69, 98, 86, 90, 78, 95, 102, 74, 83, 85, 103, 88, 87, 84, 84, 67, 77, 89, 94, 81, 80, 89, 81, 79, 86, 79, 75, 86, 84, 86, 92, 93, 87, 93, 78, 93, 66, 88, 84, 94, 65, 79, 94, 90, 100, 92, 96, 90, 99, 72, 81, 87, 90, 83, 86, 83, 99, 81, 79, 81, 82, 93, 84, 94, 95, 109, 95, 82, 98, 86, 84, 84, 69, 70, 86, 86, 89, 82, 62, 92, 81, 84, 81, 98, 81, 81, 90, 84, 70, 83, 74, 90, 85, 77, 78, 102, 91, 74, 77, 86, 77, 77, 94, 99, 104, 94, 95, 87, 81, 90, 90, 69, 80, 67, 58, 99, 74, 82, 92, 73, 104, 97, 79, 85, 75, 69, 103, 82, 78, 86, 92, 69, 80, 99, 82, 85, 85, 82, 84, 94, 88, 88, 100, 85, 94, 98, 90, 89, 97, 92, 76, 75, 102, 88, 91, 82, 87, 92, 78, 81, 81, 83, 77, 74, 77, 73, 77, 81, 86, 97, 108, 92, 71, 90, 76, 80, 85, 81, 99, 90, 79, 90, 88, 91, 93, 93, 84, 74, 82, 88, 89, 78, 80, 95, 78, 110, 83, 85, 90, 87, 89, 99, 97, 100, 86, 88, 69, 96, 86, 81, 88, 76, 65, 76, 89, 95, 81, 71, 70, 67, 90, 88, 87, 87, 84, 79, 80, 83, 98, 110, 96, 87, 89, 78, 96, 80, 76, 87, 66, 103, 103, 98, 70, 98, 89, 82, 71, 85, 81, 92, 83, 82, 94, 95, 78, 74, 76, 94, 95, 75, 90, 80, 86, 77, 72, 79, 82, 85, 84, 93, 94, 93, 82, 71, 89, 74, 81, 92, 90, 77, 85, 70, 77, 76, 86, 80, 79, 95, 98, 91, 92, 84, 88, 87, 76, 91, 79, 83, 81, 85, 74, 93, 79, 90, 79, 101, 90, 87, 77, 84, 93, 75, 85, 89, 80, 84, 91, 82, 96, 92, 73, 75, 84, 80, 98, 90, 67, 99, 80, 86, 96, 83, 93, 64, 95, 85, 88, 98, 73, 80, 81, 103, 90, 113, 80, 83] payload_rows_per_dst=[5530, 5472, 5526, 5370, 5486, 5352, 5324, 5440] dup_rows_per_dst=[2814, 2756, 2662, 2674, 2742, 2836, 2744, 2808] combine_rows_per_src=[5530, 5472, 5526, 5370, 5486, 5352, 5324, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541900800 dispatch_meta_bytes=229408 combine_payload_bytes=541900800 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542130208 ingress_bytes=718061568 link_bytes=718061568 token_bytes=1084031008 weight_movement_bytes=176160768 total_nvlink_bytes=1260191776 weight_to_token_ratio=0.162505 gemm_rows_defined=36864 gemm_rows_charged=45056 gemm_tail_pad_rows=8192 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.181818 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031267571e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541786112 dispatch_meta_bytes=229504 combine_payload_bytes=541786112 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542015616 ingress_bytes=717946880 link_bytes=717946880 token_bytes=1083801728 weight_movement_bytes=176160768 total_nvlink_bytes=1259962496 weight_to_token_ratio=0.162540 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.030304263e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543592448 dispatch_meta_bytes=229392 combine_payload_bytes=543592448 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543821840 ingress_bytes=719753216 link_bytes=719753216 token_bytes=1087414288 weight_movement_bytes=176160768 total_nvlink_bytes=1263575056 weight_to_token_ratio=0.162000 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544251904 dispatch_meta_bytes=229736 combine_payload_bytes=544251904 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=544481640 ingress_bytes=1072734208 link_bytes=1072734208 token_bytes=1088733544 weight_movement_bytes=528482304 total_nvlink_bytes=1617215848 weight_to_token_ratio=0.485410 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.193182 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.010295675e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542760960 dispatch_meta_bytes=229320 combine_payload_bytes=542760960 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=542990280 ingress_bytes=1247404032 link_bytes=1247404032 token_bytes=1085751240 weight_movement_bytes=704643072 total_nvlink_bytes=1790394312 weight_to_token_ratio=0.648991 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047741283e-04 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084031008 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542130208 ingress_bytes=718061568 link_bytes=718061568 total_nvlink_bytes=1260191776 weight_to_token_ratio=0.162505 prefetch_share_of_ingress=0.245328 dedup_ratio=0.341675 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.181818 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031267571e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083801728 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542015616 ingress_bytes=717946880 link_bytes=717946880 total_nvlink_bytes=1259962496 weight_to_token_ratio=0.162540 prefetch_share_of_ingress=0.245367 dedup_ratio=0.341553 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.030304263e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=538947584 dispatch_meta_bytes=229384 combine_payload_bytes=538947584 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=539176968 ingress_bytes=715108352 link_bytes=715108352 token_bytes=1078124552 weight_movement_bytes=176160768 total_nvlink_bytes=1254285320 weight_to_token_ratio=0.163396 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.006462405e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087414288 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543821840 ingress_bytes=719753216 link_bytes=719753216 total_nvlink_bytes=1263575056 weight_to_token_ratio=0.162000 prefetch_share_of_ingress=0.244752 dedup_ratio=0.337189 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543506432 dispatch_meta_bytes=229944 combine_payload_bytes=543506432 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543736376 ingress_bytes=719667200 link_bytes=719667200 token_bytes=1087242808 weight_movement_bytes=176160768 total_nvlink_bytes=1263403576 weight_to_token_ratio=0.162025 gemm_rows_defined=36992 gemm_rows_charged=45056 gemm_tail_pad_rows=8064 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.178977 gemm_active_groups=50 gemm_peak_group_rows=896 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.044753874e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088733544 prefetch_bytes=0 reduce_bytes=528482304 weight_movement_bytes=528482304 egress_bytes=544481640 ingress_bytes=1072734208 link_bytes=1072734208 total_nvlink_bytes=1617215848 weight_to_token_ratio=0.485410 prefetch_share_of_ingress=0.000000 dedup_ratio=0.338287 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=3 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.010295675e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085751240 prefetch_bytes=176160768 reduce_bytes=528482304 weight_movement_bytes=704643072 egress_bytes=542990280 ingress_bytes=1247404032 link_bytes=1247404032 total_nvlink_bytes=1790394312 weight_to_token_ratio=0.648991 prefetch_share_of_ingress=0.141222 dedup_ratio=0.340393 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=3 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047741283e-04 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544395264 dispatch_meta_bytes=229392 combine_payload_bytes=544395264 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=544624656 ingress_bytes=896716800 link_bytes=896716800 token_bytes=1089019920 weight_movement_bytes=352321536 total_nvlink_bytes=1441341456 weight_to_token_ratio=0.323522 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.531859658e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1078124552 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=539176968 ingress_bytes=715108352 link_bytes=715108352 total_nvlink_bytes=1254285320 weight_to_token_ratio=0.163396 prefetch_share_of_ingress=0.246341 dedup_ratio=0.344299 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.006462405e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087242808 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543736376 ingress_bytes=719667200 link_bytes=719667200 total_nvlink_bytes=1263403576 weight_to_token_ratio=0.162025 prefetch_share_of_ingress=0.244781 dedup_ratio=0.340118 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.178977 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.044753874e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1089019920 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=544624656 ingress_bytes=896716800 link_bytes=896716800 total_nvlink_bytes=1441341456 weight_to_token_ratio=0.323522 prefetch_share_of_ingress=0.196451 dedup_ratio=0.336243 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.531859658e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 121.7 | throughput per GPU (TFLOP/s/GPU): 290.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.092503E+01 | loss scale: 1.0 | grad norm: 10.970 | number of skipped iterations: 0 | number of nan iterations: 0 | +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431] sizes [768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 512, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 768, 768, 640, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 768, 640, 768, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[376, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [336, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [30, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [59, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [333, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [379, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43136 payload_rows_remote=37752 dup_rows=22400 dup_rows_remote=19566 dedup_ratio=0.341797 router_tokens_per_expert=[99, 101, 91, 80, 93, 85, 82, 99, 89, 107, 110, 75, 95, 96, 97, 104, 78, 84, 85, 91, 80, 93, 82, 85, 69, 87, 83, 89, 88, 83, 87, 79, 94, 87, 92, 104, 98, 83, 82, 90, 82, 73, 88, 76, 73, 95, 77, 95, 88, 80, 92, 79, 97, 103, 88, 73, 85, 83, 82, 93, 72, 82, 103, 77, 79, 73, 80, 88, 82, 82, 89, 100, 84, 79, 79, 85, 92, 83, 85, 71, 75, 90, 86, 85, 83, 82, 72, 85, 76, 86, 79, 82, 101, 76, 87, 92, 83, 91, 87, 84, 81, 83, 75, 92, 83, 81, 84, 82, 91, 102, 90, 79, 105, 91, 84, 100, 82, 83, 81, 105, 79, 93, 76, 82, 101, 91, 88, 88, 84, 76, 70, 91, 67, 84, 80, 66, 88, 85, 89, 84, 91, 93, 77, 87, 74, 65, 77, 82, 77, 87, 89, 92, 79, 102, 107, 77, 76, 95, 100, 92, 76, 83, 81, 86, 69, 84, 83, 74, 82, 87, 103, 74, 97, 78, 79, 89, 91, 89, 89, 81, 71, 75, 99, 94, 74, 95, 73, 103, 73, 103, 97, 71, 93, 91, 81, 82, 83, 88, 97, 79, 78, 65, 82, 80, 77, 94, 95, 77, 61, 70, 97, 81, 82, 76, 89, 70, 86, 87, 98, 82, 89, 82, 77, 68, 74, 97, 76, 84, 80, 94, 82, 99, 95, 86, 97, 96, 71, 71, 92, 98, 97, 74, 74, 75, 88, 80, 87, 72, 75, 81, 111, 110, 69, 89, 63, 87, 86, 87, 98, 74, 88, 72, 74, 75, 77, 76, 82, 82, 69, 75, 86, 98, 75, 89, 92, 109, 96, 96, 79, 70, 88, 88, 87, 63, 64, 79, 77, 84, 76, 84, 75, 86, 82, 110, 103, 90, 82, 95, 90, 74, 95, 84, 73, 73, 85, 86, 69, 76, 92, 79, 85, 84, 85, 77, 76, 88, 98, 75, 76, 93, 91, 98, 100, 110, 83, 69, 66, 90, 69, 78, 73, 73, 89, 98, 80, 83, 96, 75, 83, 83, 77, 95, 89, 91, 78, 94, 97, 95, 98, 94, 93, 96, 98, 91, 81, 90, 82, 97, 81, 97, 93, 80, 90, 95, 87, 82, 94, 78, 85, 92, 81, 91, 89, 98, 97, 68, 102, 77, 87, 96, 96, 72, 86, 96] payload_rows_per_dst=[5586, 5364, 5384, 5354, 5354, 5282, 5298, 5514] dup_rows_per_dst=[2884, 2726, 2834, 2794, 2704, 2652, 2794, 3012] combine_rows_per_src=[5586, 5364, 5384, 5354, 5354, 5282, 5298, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43306 payload_rows_remote=37796 dup_rows=22230 dup_rows_remote=19402 dedup_ratio=0.339203 router_tokens_per_expert=[79, 77, 70, 118, 82, 88, 86, 92, 84, 88, 95, 78, 112, 80, 76, 95, 99, 81, 104, 91, 81, 83, 64, 81, 92, 82, 81, 70, 71, 106, 97, 101, 84, 68, 105, 97, 76, 95, 70, 88, 82, 76, 81, 81, 88, 60, 93, 86, 90, 96, 89, 82, 87, 87, 87, 92, 87, 87, 87, 83, 75, 96, 86, 94, 76, 87, 82, 78, 89, 97, 83, 76, 75, 91, 85, 95, 98, 84, 80, 92, 93, 96, 86, 77, 94, 89, 108, 91, 67, 76, 91, 98, 72, 99, 98, 61, 86, 95, 84, 91, 91, 82, 96, 87, 83, 63, 93, 76, 69, 90, 76, 75, 79, 93, 76, 82, 69, 95, 72, 87, 87, 78, 87, 83, 96, 79, 98, 82, 89, 85, 78, 78, 90, 86, 78, 75, 91, 86, 88, 120, 86, 76, 89, 84, 105, 80, 75, 85, 89, 99, 77, 89, 73, 100, 92, 81, 89, 84, 81, 80, 73, 76, 73, 76, 89, 108, 80, 106, 92, 68, 99, 72, 77, 83, 83, 96, 79, 79, 98, 82, 81, 86, 92, 90, 82, 91, 82, 80, 90, 61, 99, 88, 73, 83, 87, 98, 76, 85, 83, 75, 85, 100, 67, 75, 81, 79, 88, 75, 95, 90, 85, 86, 81, 90, 79, 76, 73, 74, 68, 86, 83, 76, 80, 64, 88, 73, 68, 79, 78, 87, 88, 97, 101, 112, 95, 103, 89, 90, 79, 96, 81, 80, 83, 95, 87, 91, 68, 82, 69, 111, 90, 86, 78, 84, 86, 93, 99, 97, 96, 84, 96, 79, 85, 81, 102, 78, 98, 71, 86, 96, 79, 74, 77, 78, 67, 80, 82, 104, 74, 103, 81, 90, 71, 99, 103, 77, 98, 82, 86, 89, 103, 85, 99, 98, 83, 67, 67, 92, 77, 91, 70, 83, 73, 84, 87, 90, 93, 102, 91, 94, 76, 83, 85, 94, 82, 73, 83, 73, 92, 93, 75, 81, 96, 93, 81, 66, 96, 80, 89, 86, 92, 87, 79, 91, 84, 73, 81, 80, 72, 84, 87, 91, 89, 88, 89, 107, 84, 74, 85, 69, 85, 78, 85, 97, 79, 77, 77, 87, 93, 98, 78, 98, 68, 76, 90, 93, 90, 79, 91, 79, 96, 85, 89, 76, 91, 86, 81, 85, 86, 105, 84, 81, 80, 96] payload_rows_per_dst=[5412, 5510, 5428, 5360, 5358, 5462, 5376, 5400] dup_rows_per_dst=[2816, 2828, 2690, 2820, 2830, 2800, 2648, 2798] combine_rows_per_src=[5412, 5510, 5428, 5360, 5358, 5462, 5376, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43190 payload_rows_remote=37724 dup_rows=22346 dup_rows_remote=19662 dedup_ratio=0.340973 router_tokens_per_expert=[92, 78, 89, 99, 90, 91, 86, 87, 84, 98, 99, 87, 82, 92, 84, 88, 99, 102, 86, 90, 73, 84, 75, 84, 84, 80, 82, 78, 85, 81, 123, 87, 76, 77, 74, 84, 71, 95, 94, 82, 84, 81, 90, 89, 94, 65, 86, 88, 82, 91, 82, 96, 84, 76, 85, 80, 77, 76, 110, 101, 104, 89, 100, 94, 79, 70, 89, 63, 75, 82, 87, 104, 81, 74, 89, 66, 106, 85, 95, 90, 84, 84, 86, 75, 100, 77, 74, 86, 81, 84, 83, 86, 90, 90, 88, 96, 91, 78, 86, 78, 82, 98, 82, 81, 75, 89, 75, 98, 81, 91, 104, 109, 70, 81, 79, 89, 105, 73, 83, 76, 105, 86, 83, 88, 87, 94, 81, 98, 95, 83, 68, 96, 76, 82, 76, 87, 89, 84, 89, 93, 93, 98, 79, 83, 89, 84, 82, 81, 86, 83, 84, 95, 79, 81, 89, 71, 90, 80, 93, 62, 96, 82, 85, 90, 101, 87, 77, 76, 76, 76, 83, 89, 87, 81, 94, 84, 77, 68, 98, 86, 103, 92, 89, 77, 77, 88, 92, 80, 90, 79, 94, 92, 80, 98, 77, 86, 84, 103, 86, 85, 104, 75, 88, 83, 93, 89, 84, 88, 92, 84, 96, 75, 99, 89, 82, 87, 72, 80, 86, 105, 87, 92, 72, 70, 101, 90, 89, 83, 78, 88, 80, 76, 94, 97, 88, 94, 72, 79, 82, 92, 89, 92, 80, 77, 95, 69, 94, 86, 80, 74, 80, 82, 79, 84, 79, 97, 89, 72, 69, 88, 85, 69, 98, 87, 94, 91, 92, 101, 89, 88, 87, 85, 84, 70, 89, 85, 64, 96, 94, 86, 75, 75, 75, 86, 93, 72, 87, 84, 69, 84, 94, 86, 81, 88, 88, 78, 82, 86, 85, 81, 82, 86, 71, 85, 78, 72, 85, 90, 79, 88, 97, 87, 85, 91, 97, 76, 79, 59, 75, 89, 85, 80, 78, 84, 88, 80, 95, 79, 84, 85, 77, 73, 89, 104, 78, 90, 86, 85, 78, 76, 81, 74, 103, 90, 106, 93, 83, 81, 71, 98, 74, 82, 98, 93, 81, 81, 92, 76, 94, 75, 81, 83, 77, 93, 84, 89, 76, 70, 89, 65, 79, 84, 76, 89, 101, 75, 97, 70, 89, 90, 101, 112, 85, 73] payload_rows_per_dst=[5426, 5416, 5476, 5466, 5464, 5290, 5294, 5358] dup_rows_per_dst=[2872, 2836, 2818, 2684, 2844, 2782, 2710, 2800] combine_rows_per_src=[5426, 5416, 5476, 5466, 5464, 5290, 5294, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43304 payload_rows_remote=37882 dup_rows=22232 dup_rows_remote=19520 dedup_ratio=0.339233 router_tokens_per_expert=[77, 86, 86, 97, 106, 71, 69, 101, 93, 88, 92, 83, 89, 110, 87, 102, 90, 98, 98, 102, 64, 77, 94, 84, 82, 83, 92, 97, 83, 65, 92, 94, 80, 74, 85, 86, 84, 79, 89, 95, 87, 99, 98, 85, 85, 74, 87, 109, 82, 76, 69, 94, 81, 91, 83, 84, 101, 90, 97, 90, 107, 104, 72, 80, 76, 68, 82, 63, 104, 97, 67, 88, 87, 84, 83, 92, 88, 78, 87, 87, 83, 101, 77, 101, 104, 86, 77, 78, 78, 90, 81, 90, 78, 84, 96, 89, 85, 85, 93, 74, 76, 73, 90, 90, 84, 76, 83, 80, 94, 78, 94, 98, 69, 78, 102, 86, 91, 66, 99, 94, 76, 81, 87, 79, 98, 92, 92, 78, 80, 72, 100, 64, 95, 80, 81, 83, 105, 76, 82, 101, 92, 81, 86, 93, 83, 75, 88, 82, 96, 99, 81, 77, 86, 77, 96, 87, 68, 73, 106, 89, 70, 66, 93, 79, 83, 81, 85, 78, 96, 77, 102, 88, 95, 99, 78, 91, 106, 84, 83, 81, 81, 90, 89, 70, 75, 76, 80, 78, 88, 64, 94, 94, 78, 92, 94, 91, 81, 100, 86, 93, 96, 86, 87, 79, 71, 92, 95, 82, 94, 93, 75, 89, 94, 82, 78, 97, 100, 73, 74, 69, 109, 69, 82, 66, 96, 81, 69, 71, 91, 72, 81, 76, 82, 83, 65, 91, 77, 87, 86, 112, 94, 86, 86, 91, 78, 90, 92, 89, 68, 87, 88, 87, 89, 92, 101, 78, 67, 89, 65, 86, 74, 83, 87, 78, 74, 100, 85, 101, 82, 78, 87, 87, 96, 76, 105, 69, 81, 73, 91, 90, 88, 90, 82, 88, 91, 88, 97, 95, 84, 98, 102, 85, 62, 79, 91, 75, 78, 86, 84, 97, 82, 81, 92, 78, 79, 79, 80, 100, 75, 89, 98, 65, 89, 84, 82, 93, 73, 91, 80, 63, 79, 85, 89, 89, 84, 86, 77, 83, 91, 82, 83, 73, 85, 96, 83, 80, 92, 83, 73, 85, 74, 84, 86, 86, 76, 90, 74, 84, 72, 69, 86, 83, 75, 87, 81, 89, 99, 93, 72, 95, 81, 71, 87, 93, 78, 94, 81, 79, 83, 75, 86, 88, 73, 93, 75, 101, 101, 77, 83, 101, 101, 91, 81, 100] payload_rows_per_dst=[5410, 5468, 5438, 5392, 5422, 5480, 5314, 5380] dup_rows_per_dst=[2988, 2782, 2746, 2722, 2712, 2816, 2724, 2742] combine_rows_per_src=[5410, 5468, 5438, 5392, 5422, 5480, 5314, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43448 payload_rows_remote=37952 dup_rows=22088 dup_rows_remote=19308 dedup_ratio=0.337036 router_tokens_per_expert=[84, 102, 103, 82, 100, 89, 81, 72, 75, 86, 78, 80, 101, 86, 87, 90, 75, 86, 95, 75, 87, 78, 100, 94, 90, 99, 102, 80, 80, 92, 82, 79, 98, 85, 77, 77, 81, 86, 89, 78, 85, 75, 76, 86, 85, 83, 96, 90, 98, 81, 105, 87, 88, 79, 88, 96, 83, 83, 77, 104, 81, 80, 65, 88, 88, 83, 86, 88, 93, 80, 90, 79, 97, 102, 70, 68, 75, 86, 66, 99, 69, 75, 99, 82, 79, 74, 92, 80, 85, 82, 70, 77, 90, 83, 101, 83, 82, 110, 99, 89, 88, 81, 86, 81, 65, 71, 92, 99, 86, 81, 77, 91, 84, 82, 81, 86, 80, 91, 83, 76, 84, 86, 74, 77, 89, 93, 96, 81, 87, 77, 81, 94, 68, 89, 92, 72, 88, 77, 79, 97, 81, 93, 82, 71, 100, 82, 77, 86, 86, 89, 82, 83, 85, 92, 91, 75, 80, 93, 101, 90, 83, 91, 103, 93, 74, 90, 93, 91, 85, 76, 72, 65, 77, 80, 97, 101, 83, 87, 88, 86, 89, 96, 90, 75, 91, 75, 106, 84, 78, 98, 85, 75, 98, 81, 87, 78, 82, 80, 94, 98, 82, 82, 87, 73, 84, 84, 90, 101, 86, 87, 118, 90, 81, 74, 72, 55, 94, 86, 75, 101, 93, 91, 88, 81, 107, 85, 77, 68, 102, 96, 83, 72, 74, 95, 72, 85, 74, 80, 84, 77, 92, 78, 74, 75, 92, 78, 80, 81, 97, 76, 76, 72, 82, 91, 79, 106, 76, 100, 83, 96, 92, 91, 81, 74, 76, 84, 86, 94, 81, 89, 76, 77, 86, 91, 84, 88, 101, 90, 85, 75, 86, 79, 91, 89, 76, 90, 73, 79, 91, 72, 88, 89, 77, 88, 80, 87, 110, 80, 82, 64, 66, 112, 84, 85, 102, 74, 87, 87, 94, 99, 82, 77, 90, 78, 80, 80, 96, 87, 75, 103, 83, 91, 86, 78, 84, 81, 96, 75, 73, 92, 81, 76, 90, 111, 90, 76, 86, 88, 74, 94, 88, 88, 78, 94, 83, 84, 71, 93, 78, 96, 93, 90, 92, 105, 74, 101, 100, 91, 79, 63, 82, 72, 105, 82, 102, 100, 89, 82, 85, 68, 65, 86, 82, 74, 98, 85, 89, 83, 93, 87, 89, 78, 83, 96] payload_rows_per_dst=[5266, 5286, 5486, 5576, 5414, 5496, 5428, 5496] dup_rows_per_dst=[2642, 2614, 2776, 2930, 2754, 2802, 2790, 2780] combine_rows_per_src=[5266, 5286, 5486, 5576, 5414, 5496, 5428, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43390 payload_rows_remote=37924 dup_rows=22146 dup_rows_remote=19282 dedup_ratio=0.337921 router_tokens_per_expert=[81, 104, 73, 96, 93, 87, 96, 89, 78, 94, 92, 78, 100, 81, 84, 81, 86, 99, 92, 96, 96, 88, 99, 94, 77, 90, 80, 74, 93, 83, 97, 79, 88, 69, 87, 90, 99, 88, 87, 94, 79, 68, 72, 80, 97, 72, 97, 72, 84, 81, 79, 69, 97, 81, 95, 88, 107, 95, 83, 94, 95, 89, 86, 83, 90, 76, 91, 109, 96, 68, 89, 95, 91, 94, 102, 88, 63, 77, 93, 96, 97, 86, 75, 96, 82, 86, 74, 84, 79, 83, 84, 93, 92, 77, 93, 87, 104, 81, 86, 81, 87, 97, 99, 75, 70, 91, 80, 82, 82, 73, 86, 97, 87, 89, 90, 78, 80, 89, 83, 82, 78, 69, 82, 73, 93, 93, 78, 68, 79, 95, 86, 86, 94, 83, 100, 93, 95, 96, 93, 71, 68, 93, 84, 92, 97, 66, 81, 89, 75, 78, 79, 99, 88, 77, 75, 86, 91, 68, 95, 75, 83, 90, 107, 77, 86, 92, 80, 81, 98, 83, 77, 76, 67, 80, 75, 94, 83, 82, 79, 74, 85, 88, 95, 90, 77, 96, 97, 88, 83, 93, 74, 87, 66, 72, 90, 88, 71, 76, 93, 87, 96, 83, 83, 85, 87, 79, 95, 89, 95, 78, 85, 87, 98, 92, 82, 90, 77, 73, 86, 95, 66, 78, 93, 89, 82, 71, 92, 81, 99, 82, 92, 86, 94, 87, 82, 84, 90, 83, 78, 99, 66, 90, 90, 102, 114, 81, 63, 83, 94, 100, 69, 96, 75, 93, 84, 87, 80, 96, 92, 84, 72, 85, 87, 71, 96, 84, 95, 80, 83, 84, 77, 82, 69, 84, 83, 75, 90, 85, 94, 68, 89, 75, 62, 87, 99, 80, 89, 101, 91, 81, 106, 79, 86, 87, 95, 79, 87, 81, 65, 93, 81, 82, 77, 88, 103, 81, 92, 93, 84, 94, 91, 96, 98, 73, 100, 83, 69, 80, 77, 87, 87, 92, 89, 90, 86, 88, 87, 66, 91, 92, 88, 81, 81, 77, 67, 85, 99, 78, 88, 71, 73, 105, 73, 82, 78, 90, 87, 92, 88, 61, 74, 94, 86, 92, 74, 80, 78, 76, 88, 77, 74, 85, 98, 72, 88, 93, 86, 84, 88, 67, 65, 84, 102, 96, 96, 86, 83, 81, 82, 94, 74, 86, 88, 87] payload_rows_per_dst=[5372, 5462, 5446, 5446, 5378, 5466, 5424, 5396] dup_rows_per_dst=[2740, 2900, 2762, 2648, 2794, 2864, 2788, 2650] combine_rows_per_src=[5372, 5462, 5446, 5446, 5378, 5466, 5424, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43304 payload_rows_remote=37858 dup_rows=22232 dup_rows_remote=19390 dedup_ratio=0.339233 router_tokens_per_expert=[86, 86, 86, 82, 93, 75, 103, 94, 72, 79, 77, 94, 106, 89, 89, 92, 78, 84, 94, 83, 102, 77, 84, 92, 79, 92, 69, 75, 92, 76, 99, 87, 74, 76, 99, 96, 74, 78, 95, 81, 92, 85, 94, 88, 79, 70, 79, 72, 95, 100, 75, 81, 91, 95, 90, 96, 80, 94, 80, 92, 80, 86, 78, 93, 82, 80, 91, 84, 107, 70, 86, 96, 82, 89, 94, 74, 95, 92, 80, 81, 82, 84, 84, 84, 86, 89, 89, 90, 75, 61, 67, 79, 85, 76, 80, 85, 80, 81, 82, 79, 82, 81, 83, 98, 91, 67, 93, 85, 81, 81, 94, 74, 84, 103, 65, 77, 95, 71, 80, 79, 95, 79, 99, 64, 89, 80, 95, 74, 70, 92, 81, 83, 84, 76, 73, 84, 69, 92, 85, 85, 83, 85, 100, 94, 102, 81, 100, 87, 91, 77, 95, 78, 86, 104, 61, 82, 77, 89, 89, 78, 81, 89, 85, 89, 81, 68, 92, 83, 92, 84, 87, 80, 93, 97, 73, 104, 79, 78, 90, 98, 88, 87, 81, 73, 79, 98, 87, 88, 85, 92, 89, 69, 92, 89, 91, 74, 78, 87, 102, 90, 77, 74, 83, 83, 63, 96, 96, 85, 86, 82, 80, 85, 82, 97, 78, 74, 94, 79, 66, 94, 88, 73, 77, 77, 87, 79, 83, 93, 74, 81, 78, 77, 84, 77, 86, 77, 93, 89, 86, 81, 90, 91, 71, 79, 86, 99, 78, 79, 74, 95, 101, 74, 78, 83, 78, 85, 83, 88, 83, 91, 106, 83, 67, 84, 78, 80, 88, 77, 96, 87, 75, 93, 80, 87, 86, 71, 94, 80, 93, 94, 86, 82, 82, 95, 82, 92, 82, 83, 114, 86, 104, 95, 78, 105, 74, 76, 88, 92, 82, 74, 98, 85, 87, 66, 91, 89, 109, 81, 100, 92, 89, 102, 81, 83, 91, 100, 81, 81, 75, 79, 104, 83, 102, 90, 76, 84, 118, 92, 93, 83, 80, 79, 81, 102, 85, 77, 101, 73, 79, 85, 104, 90, 80, 91, 100, 89, 88, 83, 90, 110, 93, 83, 67, 82, 68, 75, 78, 74, 76, 105, 83, 106, 88, 109, 94, 87, 73, 85, 82, 92, 84, 84, 89, 81, 85, 89, 100, 74, 97, 93, 85, 81, 71, 73] payload_rows_per_dst=[5446, 5450, 5336, 5464, 5398, 5388, 5440, 5382] dup_rows_per_dst=[2842, 2794, 2658, 2768, 2800, 2750, 2964, 2656] combine_rows_per_src=[5446, 5450, 5336, 5464, 5398, 5388, 5440, 5382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541212672 dispatch_meta_bytes=229272 combine_payload_bytes=541212672 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541441944 ingress_bytes=717373440 link_bytes=717373440 token_bytes=1082654616 weight_movement_bytes=176160768 total_nvlink_bytes=1258815384 weight_to_token_ratio=0.162712 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.025487726e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541843456 dispatch_meta_bytes=228792 combine_payload_bytes=541843456 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542072248 ingress_bytes=894164992 link_bytes=894164992 token_bytes=1083915704 weight_movement_bytes=352321536 total_nvlink_bytes=1436237240 weight_to_token_ratio=0.325045 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.510426069e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540811264 dispatch_meta_bytes=229544 combine_payload_bytes=540811264 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541040808 ingress_bytes=716972032 link_bytes=716972032 token_bytes=1081852072 weight_movement_bytes=176160768 total_nvlink_bytes=1258012840 weight_to_token_ratio=0.162833 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.022116151e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543076352 dispatch_meta_bytes=229608 combine_payload_bytes=543076352 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543305960 ingress_bytes=719237120 link_bytes=719237120 token_bytes=1086382312 weight_movement_bytes=176160768 total_nvlink_bytes=1262543080 weight_to_token_ratio=0.162154 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.041141471e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544079872 dispatch_meta_bytes=229040 combine_payload_bytes=544079872 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=544308912 ingress_bytes=1072562176 link_bytes=1072562176 token_bytes=1088388784 weight_movement_bytes=528482304 total_nvlink_bytes=1616871088 weight_to_token_ratio=0.485564 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.193182 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.008850714e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543678464 dispatch_meta_bytes=228824 combine_payload_bytes=543678464 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543907288 ingress_bytes=719839232 link_bytes=719839232 token_bytes=1087585752 weight_movement_bytes=176160768 total_nvlink_bytes=1263746520 weight_to_token_ratio=0.161974 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.046198835e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1082654616 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541441944 ingress_bytes=717373440 link_bytes=717373440 total_nvlink_bytes=1258815384 weight_to_token_ratio=0.162712 prefetch_share_of_ingress=0.245564 dedup_ratio=0.341797 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.025487726e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083915704 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542072248 ingress_bytes=894164992 link_bytes=894164992 total_nvlink_bytes=1436237240 weight_to_token_ratio=0.325045 prefetch_share_of_ingress=0.197011 dedup_ratio=0.339203 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.510426069e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1081852072 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541040808 ingress_bytes=716972032 link_bytes=716972032 total_nvlink_bytes=1258012840 weight_to_token_ratio=0.162833 prefetch_share_of_ingress=0.245701 dedup_ratio=0.340973 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.022116151e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086382312 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543305960 ingress_bytes=719237120 link_bytes=719237120 total_nvlink_bytes=1262543080 weight_to_token_ratio=0.162154 prefetch_share_of_ingress=0.244927 dedup_ratio=0.339233 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.041141471e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088388784 prefetch_bytes=0 reduce_bytes=528482304 weight_movement_bytes=528482304 egress_bytes=544308912 ingress_bytes=1072562176 link_bytes=1072562176 total_nvlink_bytes=1616871088 weight_to_token_ratio=0.485564 prefetch_share_of_ingress=0.000000 dedup_ratio=0.337036 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=3 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.008850714e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087585752 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543907288 ingress_bytes=719839232 link_bytes=719839232 total_nvlink_bytes=1263746520 weight_to_token_ratio=0.161974 prefetch_share_of_ingress=0.244722 dedup_ratio=0.337921 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.046198835e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542732288 dispatch_meta_bytes=228992 combine_payload_bytes=542732288 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=542961280 ingress_bytes=1071214592 link_bytes=1071214592 token_bytes=1085693568 weight_movement_bytes=528482304 total_nvlink_bytes=1614175872 weight_to_token_ratio=0.486769 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.997531852e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085693568 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=542961280 ingress_bytes=1071214592 link_bytes=1071214592 total_nvlink_bytes=1614175872 weight_to_token_ratio=0.486769 prefetch_share_of_ingress=0.164450 dedup_ratio=0.339233 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.997531852e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43338 payload_rows_remote=37834 dup_rows=22198 dup_rows_remote=19158 dedup_ratio=0.338715 router_tokens_per_expert=[85, 89, 107, 86, 97, 78, 108, 88, 87, 77, 88, 87, 89, 78, 90, 80, 78, 88, 96, 103, 74, 88, 99, 71, 74, 77, 96, 81, 90, 92, 95, 89, 64, 79, 92, 72, 97, 79, 86, 83, 74, 93, 85, 76, 77, 62, 88, 89, 78, 95, 77, 62, 104, 86, 92, 91, 86, 104, 95, 82, 98, 83, 90, 93, 91, 78, 80, 73, 85, 92, 97, 68, 69, 94, 90, 83, 74, 70, 86, 88, 90, 94, 93, 94, 97, 94, 72, 85, 79, 90, 86, 88, 82, 89, 80, 85, 81, 97, 85, 68, 77, 84, 92, 95, 74, 82, 104, 76, 78, 80, 89, 87, 67, 93, 88, 85, 76, 86, 103, 81, 82, 89, 93, 75, 90, 77, 83, 68, 74, 87, 86, 72, 75, 88, 78, 100, 97, 75, 83, 80, 90, 86, 85, 93, 75, 82, 82, 76, 80, 81, 87, 75, 75, 73, 93, 94, 67, 81, 81, 85, 68, 88, 95, 101, 98, 88, 81, 81, 80, 86, 90, 86, 85, 77, 76, 85, 83, 86, 95, 93, 81, 77, 76, 74, 84, 78, 90, 84, 84, 74, 85, 78, 86, 81, 87, 71, 97, 100, 80, 65, 89, 71, 74, 72, 81, 83, 97, 107, 77, 96, 96, 89, 88, 99, 72, 95, 72, 92, 84, 86, 88, 81, 100, 72, 95, 82, 80, 74, 102, 87, 82, 94, 80, 84, 87, 84, 101, 98, 90, 107, 94, 78, 81, 78, 95, 98, 89, 89, 80, 78, 87, 74, 72, 91, 103, 88, 72, 103, 85, 75, 65, 90, 70, 80, 73, 90, 74, 86, 84, 88, 85, 64, 90, 74, 81, 81, 96, 85, 99, 89, 78, 84, 78, 82, 94, 80, 75, 89, 76, 96, 86, 76, 83, 83, 78, 89, 73, 99, 90, 80, 94, 84, 106, 81, 102, 81, 85, 84, 94, 95, 106, 107, 90, 84, 89, 112, 84, 97, 97, 83, 91, 88, 94, 95, 84, 85, 78, 92, 83, 103, 77, 100, 87, 92, 86, 73, 103, 83, 82, 76, 107, 91, 85, 83, 81, 95, 107, 74, 79, 84, 88, 77, 91, 63, 72, 76, 85, 76, 82, 85, 82, 90, 82, 92, 85, 87, 75, 93, 86, 86, 102, 81, 89, 74, 82, 94, 100, 73, 84, 103, 83, 68, 83, 87] payload_rows_per_dst=[5240, 5426, 5498, 5378, 5464, 5456, 5504, 5372] dup_rows_per_dst=[2594, 2674, 2760, 2734, 2846, 2750, 3040, 2800] combine_rows_per_src=[5240, 5426, 5498, 5378, 5464, 5456, 5504, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542388224 dispatch_meta_bytes=227968 combine_payload_bytes=542388224 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542616192 ingress_bytes=894709760 link_bytes=894709760 token_bytes=1085004416 weight_movement_bytes=352321536 total_nvlink_bytes=1437325952 weight_to_token_ratio=0.324719 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.515001779e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085004416 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542616192 ingress_bytes=894709760 link_bytes=894709760 total_nvlink_bytes=1437325952 weight_to_token_ratio=0.324719 prefetch_share_of_ingress=0.196892 dedup_ratio=0.338715 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.515001779e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 123.2 | throughput per GPU (TFLOP/s/GPU): 287.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.476386E+01 | loss scale: 1.0 | grad norm: 74.312 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43008 payload_rows_remote=37530 dup_rows=22528 dup_rows_remote=19668 dedup_ratio=0.343750 router_tokens_per_expert=[81, 82, 81, 93, 88, 81, 90, 84, 95, 88, 68, 95, 88, 93, 65, 64, 86, 88, 78, 99, 98, 86, 83, 79, 83, 76, 77, 70, 85, 89, 93, 69, 87, 74, 94, 92, 108, 91, 91, 74, 97, 91, 76, 80, 92, 94, 98, 85, 96, 74, 95, 79, 84, 95, 81, 69, 85, 68, 93, 74, 84, 89, 79, 101, 84, 79, 78, 101, 90, 80, 90, 80, 69, 88, 91, 78, 94, 96, 82, 94, 88, 87, 88, 80, 77, 80, 99, 91, 70, 90, 69, 93, 89, 84, 83, 71, 75, 89, 68, 95, 89, 80, 85, 84, 85, 92, 73, 84, 77, 98, 73, 73, 78, 66, 81, 94, 90, 90, 86, 83, 68, 82, 94, 98, 95, 81, 80, 106, 94, 84, 105, 87, 93, 107, 56, 94, 79, 95, 88, 99, 68, 86, 93, 86, 76, 80, 88, 108, 90, 85, 93, 75, 102, 100, 84, 90, 98, 99, 94, 93, 87, 76, 69, 70, 86, 88, 98, 74, 81, 93, 91, 64, 76, 76, 103, 101, 91, 87, 106, 93, 88, 97, 86, 62, 94, 77, 65, 101, 90, 93, 88, 81, 87, 87, 69, 75, 93, 78, 81, 93, 106, 85, 78, 82, 81, 79, 81, 97, 80, 92, 95, 75, 99, 74, 84, 84, 88, 91, 62, 78, 77, 78, 103, 82, 94, 104, 89, 88, 99, 87, 93, 72, 104, 85, 73, 98, 98, 83, 107, 101, 87, 81, 75, 77, 91, 75, 88, 89, 92, 79, 79, 93, 86, 72, 92, 66, 90, 72, 78, 104, 100, 83, 85, 80, 80, 96, 78, 99, 78, 85, 83, 77, 83, 71, 80, 83, 71, 80, 84, 91, 91, 96, 86, 78, 91, 96, 85, 69, 81, 82, 89, 73, 76, 92, 87, 94, 91, 86, 79, 86, 96, 66, 87, 78, 96, 83, 85, 82, 89, 73, 91, 87, 76, 86, 86, 71, 91, 93, 87, 96, 72, 85, 82, 89, 87, 83, 87, 77, 70, 89, 90, 69, 95, 79, 82, 73, 82, 78, 81, 99, 103, 109, 87, 94, 87, 77, 84, 95, 82, 76, 80, 80, 83, 84, 94, 79, 95, 78, 91, 92, 80, 73, 63, 91, 94, 84, 77, 88, 94, 80, 78, 93, 83, 84, 75, 90, 78, 101, 93, 74, 88, 91, 81, 76] payload_rows_per_dst=[5372, 5348, 5346, 5456, 5478, 5238, 5346, 5424] dup_rows_per_dst=[2826, 2770, 2866, 2918, 2860, 2628, 2886, 2774] combine_rows_per_src=[5372, 5348, 5346, 5456, 5478, 5238, 5346, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43294 payload_rows_remote=37848 dup_rows=22242 dup_rows_remote=19354 dedup_ratio=0.339386 router_tokens_per_expert=[86, 88, 90, 78, 92, 70, 78, 93, 115, 88, 94, 89, 91, 83, 84, 88, 93, 96, 87, 77, 84, 78, 102, 77, 87, 95, 97, 80, 79, 99, 87, 92, 80, 78, 102, 93, 95, 72, 87, 89, 84, 78, 69, 91, 95, 81, 111, 77, 98, 68, 87, 91, 101, 89, 87, 74, 91, 90, 99, 92, 69, 96, 92, 70, 90, 67, 76, 74, 87, 93, 67, 92, 81, 78, 91, 76, 83, 97, 87, 74, 89, 89, 84, 84, 85, 74, 94, 82, 83, 82, 76, 91, 77, 59, 100, 95, 63, 102, 100, 91, 85, 81, 91, 80, 82, 77, 93, 97, 75, 94, 71, 91, 85, 92, 84, 90, 74, 90, 98, 78, 70, 102, 94, 76, 89, 86, 81, 87, 100, 99, 83, 101, 80, 103, 76, 83, 70, 82, 94, 67, 86, 82, 84, 80, 87, 82, 95, 95, 88, 93, 83, 99, 89, 93, 96, 92, 98, 82, 83, 71, 88, 79, 85, 90, 83, 91, 115, 89, 108, 104, 68, 89, 72, 85, 86, 87, 83, 85, 92, 99, 89, 85, 95, 82, 91, 88, 71, 90, 84, 81, 83, 93, 81, 82, 62, 77, 81, 75, 78, 86, 76, 91, 84, 84, 79, 100, 87, 79, 85, 83, 86, 85, 63, 96, 84, 84, 90, 91, 85, 74, 86, 84, 87, 81, 92, 83, 79, 84, 86, 84, 91, 83, 103, 83, 77, 94, 88, 93, 92, 89, 75, 99, 87, 100, 78, 83, 74, 98, 79, 73, 84, 90, 85, 95, 92, 95, 101, 83, 95, 91, 100, 95, 85, 78, 78, 84, 83, 71, 81, 91, 82, 82, 59, 97, 92, 75, 96, 87, 74, 88, 87, 98, 83, 83, 93, 78, 93, 77, 80, 81, 92, 88, 77, 82, 81, 100, 99, 85, 77, 86, 77, 85, 114, 91, 74, 74, 91, 70, 84, 85, 84, 76, 79, 81, 71, 78, 72, 82, 91, 84, 89, 90, 97, 77, 73, 82, 76, 77, 80, 77, 89, 83, 88, 76, 78, 88, 87, 99, 88, 86, 71, 86, 100, 81, 86, 83, 85, 77, 78, 78, 89, 86, 91, 84, 91, 82, 72, 70, 89, 82, 85, 71, 79, 83, 87, 81, 90, 89, 71, 94, 81, 86, 84, 84, 85, 96, 72, 69, 83, 84, 85, 59, 100, 79] payload_rows_per_dst=[5510, 5372, 5420, 5446, 5426, 5432, 5376, 5312] dup_rows_per_dst=[2888, 2868, 2818, 2888, 2668, 2760, 2668, 2684] combine_rows_per_src=[5510, 5372, 5420, 5446, 5426, 5432, 5376, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43394 payload_rows_remote=37982 dup_rows=22142 dup_rows_remote=19504 dedup_ratio=0.337860 router_tokens_per_expert=[75, 94, 99, 86, 97, 99, 87, 75, 93, 75, 80, 76, 93, 93, 84, 79, 77, 95, 86, 80, 81, 87, 95, 75, 88, 92, 93, 77, 68, 98, 78, 79, 89, 101, 81, 96, 87, 92, 71, 89, 98, 84, 84, 87, 85, 89, 94, 69, 81, 94, 92, 100, 97, 74, 80, 73, 74, 76, 90, 81, 90, 94, 92, 89, 82, 80, 97, 86, 82, 78, 76, 90, 85, 101, 100, 75, 78, 83, 97, 72, 79, 89, 89, 89, 82, 107, 82, 79, 73, 90, 87, 78, 73, 84, 95, 92, 89, 77, 85, 77, 79, 77, 70, 90, 93, 74, 92, 76, 94, 82, 79, 73, 93, 79, 94, 94, 77, 89, 104, 100, 82, 83, 95, 83, 83, 77, 89, 88, 90, 92, 82, 102, 66, 82, 81, 95, 89, 89, 83, 86, 93, 79, 91, 77, 77, 86, 104, 77, 96, 81, 83, 94, 94, 90, 82, 74, 92, 92, 74, 69, 88, 93, 73, 78, 96, 76, 96, 74, 81, 74, 86, 80, 90, 81, 90, 81, 87, 84, 96, 84, 77, 94, 72, 85, 81, 84, 79, 86, 80, 104, 75, 92, 84, 98, 81, 71, 76, 77, 93, 91, 96, 83, 79, 95, 66, 77, 76, 96, 98, 83, 97, 82, 89, 67, 85, 80, 84, 89, 79, 80, 90, 83, 80, 81, 95, 82, 83, 94, 83, 90, 71, 93, 80, 79, 84, 82, 72, 98, 86, 68, 85, 99, 76, 88, 71, 80, 81, 81, 100, 89, 101, 75, 69, 88, 94, 84, 84, 73, 82, 98, 80, 99, 96, 83, 85, 88, 75, 95, 89, 89, 81, 77, 80, 80, 91, 99, 77, 94, 74, 97, 96, 94, 84, 69, 78, 91, 91, 89, 85, 72, 81, 93, 92, 87, 84, 96, 82, 90, 99, 73, 83, 79, 72, 82, 79, 81, 80, 65, 80, 89, 82, 83, 84, 81, 80, 90, 79, 91, 101, 94, 86, 77, 82, 84, 70, 77, 68, 94, 78, 76, 75, 73, 83, 91, 78, 84, 73, 95, 69, 93, 83, 99, 106, 78, 101, 100, 85, 78, 78, 91, 74, 100, 83, 88, 84, 101, 88, 93, 74, 86, 85, 88, 93, 74, 69, 88, 95, 96, 101, 86, 100, 91, 84, 93, 81, 93, 86, 81, 97, 126, 81, 86, 102, 89] payload_rows_per_dst=[5472, 5376, 5400, 5402, 5358, 5412, 5430, 5544] dup_rows_per_dst=[2788, 2838, 2788, 2722, 2694, 2638, 2688, 2986] combine_rows_per_src=[5472, 5376, 5400, 5402, 5358, 5412, 5430, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43222 payload_rows_remote=37798 dup_rows=22314 dup_rows_remote=19458 dedup_ratio=0.340485 router_tokens_per_expert=[87, 79, 83, 86, 90, 90, 69, 90, 87, 103, 85, 94, 87, 93, 90, 82, 75, 74, 87, 75, 85, 78, 87, 89, 75, 84, 67, 70, 80, 88, 99, 77, 102, 75, 88, 79, 91, 60, 99, 87, 100, 92, 89, 82, 87, 81, 94, 75, 74, 82, 76, 86, 62, 86, 74, 85, 76, 87, 75, 75, 84, 82, 83, 83, 84, 92, 82, 86, 81, 88, 72, 93, 89, 107, 91, 83, 91, 76, 86, 83, 65, 94, 77, 90, 83, 88, 71, 96, 93, 85, 74, 94, 72, 99, 107, 100, 75, 65, 107, 92, 72, 91, 90, 70, 80, 85, 87, 85, 103, 79, 93, 66, 70, 66, 84, 84, 84, 91, 76, 100, 73, 83, 91, 105, 80, 93, 73, 90, 121, 67, 93, 87, 98, 100, 75, 81, 84, 95, 92, 94, 94, 94, 89, 91, 92, 85, 77, 91, 98, 81, 80, 90, 90, 78, 86, 91, 75, 85, 96, 74, 80, 77, 87, 99, 95, 88, 72, 76, 77, 85, 78, 75, 87, 78, 98, 85, 94, 82, 87, 88, 72, 70, 92, 81, 78, 97, 70, 85, 97, 80, 89, 109, 86, 69, 81, 83, 94, 80, 80, 77, 80, 80, 74, 75, 80, 88, 72, 66, 91, 85, 78, 83, 87, 90, 80, 80, 82, 84, 87, 82, 67, 76, 89, 82, 96, 70, 89, 75, 91, 85, 90, 77, 92, 72, 94, 87, 101, 95, 74, 91, 72, 80, 97, 108, 77, 78, 93, 80, 94, 72, 97, 103, 74, 91, 104, 92, 90, 73, 82, 71, 74, 98, 87, 99, 88, 77, 90, 96, 78, 87, 99, 74, 96, 89, 104, 88, 77, 93, 81, 96, 85, 78, 76, 88, 87, 86, 91, 78, 85, 90, 92, 97, 85, 87, 86, 90, 93, 93, 82, 105, 85, 97, 86, 75, 95, 104, 72, 79, 83, 86, 81, 97, 91, 92, 71, 75, 82, 84, 76, 81, 84, 90, 85, 72, 80, 75, 79, 77, 71, 76, 97, 91, 91, 78, 86, 97, 89, 96, 95, 99, 94, 91, 97, 71, 84, 78, 88, 87, 105, 95, 101, 89, 79, 92, 71, 93, 91, 87, 98, 70, 79, 78, 77, 89, 89, 73, 95, 85, 100, 77, 101, 87, 86, 103, 83, 101, 76, 82, 94, 89, 72, 79, 90, 79] payload_rows_per_dst=[5348, 5424, 5492, 5252, 5366, 5456, 5438, 5446] dup_rows_per_dst=[2784, 2856, 2784, 2706, 2692, 2880, 2774, 2838] combine_rows_per_src=[5348, 5424, 5492, 5252, 5366, 5456, 5438, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43180 payload_rows_remote=37722 dup_rows=22356 dup_rows_remote=19436 dedup_ratio=0.341125 router_tokens_per_expert=[98, 96, 81, 97, 81, 86, 78, 71, 102, 91, 75, 88, 86, 89, 97, 92, 88, 81, 80, 73, 98, 86, 83, 84, 96, 104, 101, 88, 77, 90, 92, 81, 103, 88, 81, 88, 95, 97, 100, 88, 97, 93, 85, 79, 80, 75, 88, 81, 78, 84, 90, 83, 88, 88, 80, 76, 86, 66, 66, 75, 85, 88, 94, 73, 86, 91, 84, 92, 111, 87, 56, 86, 72, 84, 74, 83, 58, 93, 93, 73, 72, 91, 74, 81, 71, 80, 77, 79, 75, 70, 98, 88, 72, 87, 93, 81, 76, 67, 97, 77, 105, 80, 84, 89, 91, 84, 73, 105, 80, 101, 75, 82, 70, 99, 79, 97, 95, 77, 92, 96, 80, 97, 91, 87, 56, 94, 82, 82, 73, 82, 100, 85, 74, 64, 74, 82, 71, 69, 111, 77, 86, 71, 107, 66, 66, 88, 78, 79, 95, 75, 89, 104, 76, 74, 86, 71, 98, 89, 70, 88, 81, 83, 79, 78, 92, 76, 76, 83, 78, 90, 92, 84, 97, 88, 98, 93, 99, 97, 73, 66, 85, 93, 106, 99, 75, 84, 83, 88, 101, 94, 96, 88, 100, 90, 98, 91, 85, 100, 87, 104, 87, 83, 105, 88, 83, 86, 89, 81, 79, 75, 71, 76, 79, 85, 80, 81, 90, 86, 95, 84, 83, 77, 104, 87, 73, 98, 78, 79, 96, 63, 67, 80, 95, 86, 79, 78, 86, 64, 86, 101, 80, 82, 88, 87, 86, 82, 74, 74, 73, 75, 91, 89, 85, 83, 88, 92, 74, 89, 100, 78, 99, 101, 76, 103, 88, 100, 74, 98, 97, 91, 91, 95, 93, 92, 96, 75, 78, 92, 92, 90, 86, 86, 81, 74, 78, 94, 88, 77, 71, 77, 90, 91, 86, 79, 89, 97, 80, 87, 94, 89, 92, 81, 75, 82, 86, 92, 78, 88, 83, 95, 92, 82, 80, 87, 71, 89, 76, 83, 86, 78, 82, 83, 77, 99, 77, 94, 82, 105, 79, 65, 93, 82, 106, 80, 96, 95, 81, 94, 100, 93, 86, 97, 93, 85, 88, 89, 84, 85, 78, 88, 80, 82, 96, 92, 89, 101, 81, 85, 70, 81, 65, 79, 80, 62, 73, 76, 84, 85, 84, 76, 97, 84, 101, 101, 79, 99, 94, 89, 75, 79, 78, 89, 98, 96] payload_rows_per_dst=[5508, 5340, 5318, 5372, 5348, 5376, 5458, 5460] dup_rows_per_dst=[2880, 2484, 2814, 2870, 2848, 2758, 2920, 2782] combine_rows_per_src=[5508, 5340, 5318, 5372, 5348, 5376, 5458, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43356 payload_rows_remote=38044 dup_rows=22180 dup_rows_remote=19498 dedup_ratio=0.338440 router_tokens_per_expert=[78, 91, 73, 72, 78, 82, 100, 75, 74, 71, 93, 84, 71, 84, 98, 75, 90, 84, 83, 87, 94, 85, 70, 81, 89, 96, 72, 84, 95, 83, 82, 86, 63, 97, 82, 77, 114, 80, 91, 75, 82, 82, 97, 88, 87, 96, 85, 80, 82, 69, 84, 89, 84, 84, 74, 82, 87, 84, 87, 83, 79, 84, 90, 67, 97, 90, 94, 73, 93, 80, 88, 85, 103, 76, 79, 81, 110, 75, 94, 84, 95, 75, 80, 77, 97, 85, 72, 94, 91, 111, 78, 97, 79, 96, 78, 93, 94, 75, 80, 70, 79, 82, 60, 90, 81, 71, 83, 81, 62, 84, 75, 73, 82, 96, 80, 76, 84, 100, 91, 92, 80, 70, 107, 88, 86, 82, 73, 80, 99, 102, 93, 92, 71, 111, 89, 85, 92, 92, 61, 76, 73, 97, 87, 70, 94, 78, 86, 75, 71, 106, 81, 85, 88, 99, 98, 81, 91, 85, 77, 100, 86, 80, 96, 76, 84, 98, 71, 96, 93, 71, 114, 85, 80, 75, 108, 86, 117, 90, 83, 94, 69, 91, 94, 91, 88, 77, 88, 78, 68, 69, 92, 65, 82, 75, 66, 85, 90, 69, 85, 87, 93, 77, 84, 84, 84, 79, 94, 78, 75, 86, 79, 76, 79, 95, 90, 78, 83, 72, 88, 89, 75, 93, 124, 81, 83, 73, 85, 84, 76, 96, 93, 93, 86, 85, 69, 87, 100, 91, 82, 80, 97, 81, 96, 77, 82, 86, 84, 106, 70, 78, 106, 112, 83, 76, 89, 87, 90, 78, 91, 79, 90, 87, 90, 89, 95, 74, 86, 78, 84, 85, 77, 90, 71, 86, 79, 88, 83, 86, 93, 84, 80, 95, 92, 80, 80, 88, 81, 87, 83, 77, 90, 84, 103, 73, 81, 86, 93, 87, 86, 79, 104, 78, 87, 86, 90, 83, 92, 78, 81, 80, 76, 79, 91, 79, 89, 106, 107, 85, 90, 98, 88, 62, 84, 74, 85, 85, 88, 95, 88, 82, 73, 100, 98, 76, 102, 76, 97, 87, 113, 90, 80, 98, 80, 65, 77, 80, 78, 89, 84, 82, 92, 76, 92, 89, 86, 85, 96, 91, 92, 98, 74, 88, 89, 101, 94, 81, 84, 72, 88, 86, 93, 96, 96, 71, 97, 90, 76, 67, 83, 111, 68, 110, 97, 68] payload_rows_per_dst=[5320, 5522, 5312, 5434, 5388, 5422, 5474, 5484] dup_rows_per_dst=[2752, 2912, 2682, 2646, 2688, 2830, 2800, 2870] combine_rows_per_src=[5320, 5522, 5312, 5434, 5388, 5422, 5474, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43388 payload_rows_remote=37944 dup_rows=22148 dup_rows_remote=19384 dedup_ratio=0.337952 router_tokens_per_expert=[79, 83, 87, 102, 89, 105, 95, 101, 82, 93, 96, 89, 92, 94, 75, 91, 71, 72, 84, 82, 100, 84, 86, 82, 104, 89, 82, 99, 85, 72, 77, 82, 87, 94, 92, 84, 93, 71, 84, 84, 72, 87, 81, 96, 84, 82, 85, 76, 90, 77, 72, 88, 85, 81, 106, 67, 83, 90, 96, 84, 74, 83, 85, 82, 94, 83, 82, 79, 87, 91, 82, 80, 108, 98, 107, 82, 95, 99, 75, 87, 78, 90, 64, 82, 76, 89, 75, 72, 85, 81, 83, 84, 69, 76, 84, 90, 88, 70, 86, 82, 105, 75, 73, 86, 65, 76, 80, 99, 87, 93, 75, 99, 90, 80, 102, 81, 73, 85, 91, 91, 78, 82, 80, 78, 78, 71, 77, 75, 93, 94, 108, 82, 86, 89, 100, 93, 77, 91, 81, 84, 90, 92, 106, 66, 102, 89, 85, 78, 92, 86, 67, 81, 83, 84, 78, 108, 77, 84, 81, 84, 93, 75, 76, 76, 85, 87, 83, 98, 94, 82, 107, 87, 82, 93, 91, 89, 82, 100, 91, 80, 88, 109, 89, 83, 85, 87, 81, 84, 75, 105, 91, 76, 103, 75, 100, 107, 90, 93, 90, 93, 85, 84, 92, 83, 82, 72, 90, 86, 91, 86, 86, 94, 76, 94, 76, 83, 84, 76, 63, 77, 90, 84, 86, 73, 101, 97, 71, 90, 83, 82, 84, 76, 89, 94, 81, 62, 81, 75, 99, 73, 74, 77, 97, 85, 86, 101, 82, 88, 84, 82, 83, 87, 88, 82, 107, 98, 94, 83, 81, 109, 83, 95, 91, 75, 78, 81, 80, 93, 66, 90, 91, 90, 90, 77, 85, 81, 85, 77, 91, 72, 95, 89, 100, 94, 98, 85, 111, 89, 76, 90, 86, 91, 84, 73, 87, 79, 73, 88, 81, 79, 79, 90, 75, 80, 78, 99, 80, 87, 78, 76, 94, 75, 86, 78, 73, 74, 83, 79, 74, 108, 69, 61, 104, 78, 84, 82, 75, 79, 83, 63, 84, 81, 85, 79, 95, 95, 87, 88, 95, 100, 78, 92, 87, 83, 76, 92, 87, 83, 102, 93, 71, 92, 99, 74, 82, 84, 93, 98, 86, 76, 93, 67, 94, 69, 88, 88, 88, 74, 86, 81, 99, 84, 76, 84, 90, 78, 83, 83, 92, 97, 78, 80, 83, 71] payload_rows_per_dst=[5340, 5388, 5504, 5496, 5454, 5440, 5322, 5444] dup_rows_per_dst=[2638, 2712, 2848, 2978, 2710, 2746, 2752, 2764] combine_rows_per_src=[5340, 5388, 5504, 5496, 5454, 5440, 5322, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542588928 dispatch_meta_bytes=228808 combine_payload_bytes=542588928 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542817736 ingress_bytes=894910464 link_bytes=894910464 token_bytes=1085406664 weight_movement_bytes=352321536 total_nvlink_bytes=1437728200 weight_to_token_ratio=0.324599 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.516687567e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=538030080 dispatch_meta_bytes=228792 combine_payload_bytes=538030080 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=538258872 ingress_bytes=714190848 link_bytes=714190848 token_bytes=1076288952 weight_movement_bytes=176160768 total_nvlink_bytes=1252449720 weight_to_token_ratio=0.163674 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=5.998755946e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541872128 dispatch_meta_bytes=229024 combine_payload_bytes=541872128 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542101152 ingress_bytes=718032896 link_bytes=718032896 token_bytes=1083973280 weight_movement_bytes=176160768 total_nvlink_bytes=1260134048 weight_to_token_ratio=0.162514 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544509952 dispatch_meta_bytes=229944 combine_payload_bytes=544509952 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=544739896 ingress_bytes=896831488 link_bytes=896831488 token_bytes=1089249848 weight_movement_bytes=352321536 total_nvlink_bytes=1441571384 weight_to_token_ratio=0.323453 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.532822965e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540782592 dispatch_meta_bytes=228632 combine_payload_bytes=540782592 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541011224 ingress_bytes=716943360 link_bytes=716943360 token_bytes=1081793816 weight_movement_bytes=176160768 total_nvlink_bytes=1257954584 weight_to_token_ratio=0.162841 gemm_rows_defined=36736 gemm_rows_charged=45056 gemm_tail_pad_rows=8320 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.184659 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.021875324e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=545398784 dispatch_meta_bytes=230168 combine_payload_bytes=545398784 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=545628952 ingress_bytes=721559552 link_bytes=721559552 token_bytes=1091027736 weight_movement_bytes=176160768 total_nvlink_bytes=1267188504 weight_to_token_ratio=0.161463 gemm_rows_defined=35840 gemm_rows_charged=45056 gemm_tail_pad_rows=9216 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.204545 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.060648446e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543965184 dispatch_meta_bytes=229312 combine_payload_bytes=543965184 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=544194496 ingress_bytes=1072447488 link_bytes=1072447488 token_bytes=1088159680 weight_movement_bytes=528482304 total_nvlink_bytes=1616641984 weight_to_token_ratio=0.485666 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.198864 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.007887407e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085406664 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542817736 ingress_bytes=894910464 link_bytes=894910464 total_nvlink_bytes=1437728200 weight_to_token_ratio=0.324599 prefetch_share_of_ingress=0.196847 dedup_ratio=0.339386 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.516687567e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43238 payload_rows_remote=37880 dup_rows=22298 dup_rows_remote=19546 dedup_ratio=0.340240 router_tokens_per_expert=[94, 71, 84, 76, 78, 79, 89, 90, 88, 88, 81, 72, 84, 98, 83, 80, 81, 101, 80, 92, 82, 93, 86, 72, 79, 106, 81, 82, 85, 95, 78, 93, 87, 78, 96, 92, 96, 86, 81, 72, 93, 89, 67, 85, 81, 81, 83, 65, 83, 70, 92, 88, 91, 84, 82, 75, 87, 79, 75, 80, 92, 80, 97, 92, 91, 77, 82, 94, 94, 86, 67, 95, 101, 81, 84, 82, 97, 89, 87, 100, 69, 82, 79, 92, 76, 85, 91, 88, 73, 91, 80, 82, 76, 87, 79, 81, 97, 77, 88, 84, 103, 70, 79, 83, 82, 86, 94, 104, 85, 90, 87, 77, 83, 80, 87, 99, 99, 95, 88, 95, 89, 83, 78, 73, 82, 93, 90, 85, 88, 91, 79, 99, 80, 90, 81, 84, 71, 82, 73, 72, 68, 75, 84, 70, 81, 77, 95, 87, 96, 97, 74, 89, 88, 80, 78, 94, 73, 77, 80, 74, 66, 97, 94, 76, 83, 85, 75, 76, 90, 82, 81, 91, 87, 82, 98, 94, 94, 85, 91, 77, 93, 79, 99, 81, 91, 71, 75, 90, 86, 81, 90, 93, 95, 89, 79, 80, 82, 86, 84, 88, 90, 123, 88, 88, 81, 82, 82, 80, 101, 88, 90, 83, 100, 96, 87, 92, 79, 87, 86, 105, 72, 100, 79, 73, 86, 103, 103, 87, 80, 72, 86, 87, 64, 74, 88, 103, 99, 76, 85, 82, 90, 98, 85, 89, 92, 92, 82, 113, 78, 80, 75, 80, 64, 92, 83, 87, 84, 71, 95, 83, 97, 89, 89, 86, 90, 86, 96, 73, 85, 84, 81, 105, 90, 90, 93, 79, 76, 93, 74, 91, 84, 92, 85, 96, 76, 94, 101, 74, 94, 98, 75, 82, 76, 88, 80, 78, 77, 101, 88, 96, 77, 79, 85, 91, 83, 86, 90, 102, 72, 86, 87, 73, 93, 83, 85, 71, 74, 94, 90, 84, 100, 79, 88, 80, 76, 79, 76, 85, 94, 73, 81, 107, 89, 93, 90, 94, 89, 104, 88, 81, 86, 89, 88, 80, 93, 70, 91, 73, 92, 94, 81, 76, 77, 76, 82, 82, 70, 90, 87, 81, 79, 95, 86, 80, 86, 91, 81, 74, 90, 70, 106, 99, 62, 89, 93, 86, 98, 81, 82, 72, 76, 88, 93, 74] payload_rows_per_dst=[5358, 5466, 5404, 5302, 5488, 5552, 5438, 5230] dup_rows_per_dst=[2752, 2860, 2740, 2648, 3070, 2968, 2766, 2494] combine_rows_per_src=[5358, 5466, 5404, 5302, 5488, 5552, 5438, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1076288952 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=538258872 ingress_bytes=714190848 link_bytes=714190848 total_nvlink_bytes=1252449720 weight_to_token_ratio=0.163674 prefetch_share_of_ingress=0.246658 dedup_ratio=0.343750 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=5.998755946e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083973280 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542101152 ingress_bytes=718032896 link_bytes=718032896 total_nvlink_bytes=1260134048 weight_to_token_ratio=0.162514 prefetch_share_of_ingress=0.245338 dedup_ratio=0.340485 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1081793816 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541011224 ingress_bytes=716943360 link_bytes=716943360 total_nvlink_bytes=1257954584 weight_to_token_ratio=0.162841 prefetch_share_of_ingress=0.245711 dedup_ratio=0.341125 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.184659 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.021875324e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1091027736 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=545628952 ingress_bytes=721559552 link_bytes=721559552 total_nvlink_bytes=1267188504 weight_to_token_ratio=0.161463 prefetch_share_of_ingress=0.244139 dedup_ratio=0.338440 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.204545 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.060648446e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1089249848 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=544739896 ingress_bytes=896831488 link_bytes=896831488 total_nvlink_bytes=1441571384 weight_to_token_ratio=0.323453 prefetch_share_of_ingress=0.196426 dedup_ratio=0.337860 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.532822965e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088159680 prefetch_bytes=0 reduce_bytes=528482304 weight_movement_bytes=528482304 egress_bytes=544194496 ingress_bytes=1072447488 link_bytes=1072447488 total_nvlink_bytes=1616641984 weight_to_token_ratio=0.485666 prefetch_share_of_ingress=0.000000 dedup_ratio=0.337952 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=3 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.007887407e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543047680 dispatch_meta_bytes=229704 combine_payload_bytes=543047680 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=543277384 ingress_bytes=1071529984 link_bytes=1071529984 token_bytes=1086325064 weight_movement_bytes=528482304 total_nvlink_bytes=1614807368 weight_to_token_ratio=0.486486 gemm_rows_defined=36736 gemm_rows_charged=45056 gemm_tail_pad_rows=8320 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.184659 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.000180948e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086325064 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=543277384 ingress_bytes=1071529984 link_bytes=1071529984 total_nvlink_bytes=1614807368 weight_to_token_ratio=0.486486 prefetch_share_of_ingress=0.164401 dedup_ratio=0.340240 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.184659 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.000180948e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 120.1 | throughput per GPU (TFLOP/s/GPU): 294.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.822297E+01 | loss scale: 1.0 | grad norm: 104.016 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43100 payload_rows_remote=37636 dup_rows=22436 dup_rows_remote=19670 dedup_ratio=0.342346 router_tokens_per_expert=[94, 78, 74, 80, 102, 78, 88, 84, 87, 82, 81, 74, 79, 87, 87, 79, 105, 90, 87, 81, 83, 75, 82, 97, 90, 81, 93, 75, 76, 93, 80, 92, 79, 75, 94, 88, 78, 88, 79, 81, 106, 96, 89, 75, 89, 99, 75, 63, 111, 118, 83, 71, 70, 79, 65, 85, 60, 97, 91, 76, 74, 87, 71, 62, 71, 114, 85, 85, 84, 82, 87, 95, 91, 73, 89, 96, 78, 65, 75, 87, 88, 87, 98, 84, 74, 91, 81, 77, 88, 81, 81, 81, 71, 87, 90, 79, 101, 77, 73, 74, 95, 88, 88, 87, 84, 67, 91, 74, 97, 85, 97, 89, 85, 84, 88, 79, 102, 96, 80, 101, 85, 83, 73, 85, 81, 81, 91, 99, 103, 94, 78, 83, 92, 88, 75, 107, 82, 90, 87, 93, 90, 89, 93, 76, 82, 80, 78, 88, 92, 83, 85, 66, 87, 82, 77, 73, 72, 99, 89, 86, 83, 76, 106, 93, 68, 64, 74, 60, 72, 96, 70, 87, 89, 86, 86, 95, 77, 108, 73, 99, 86, 90, 100, 86, 79, 72, 84, 85, 85, 87, 86, 87, 91, 84, 103, 97, 92, 92, 82, 102, 78, 74, 84, 92, 80, 92, 100, 97, 94, 99, 71, 98, 79, 93, 85, 77, 91, 94, 89, 106, 87, 96, 85, 90, 80, 81, 77, 85, 80, 96, 96, 98, 80, 80, 96, 89, 67, 96, 93, 84, 87, 95, 67, 90, 95, 95, 85, 72, 99, 82, 87, 85, 96, 80, 88, 85, 76, 100, 80, 90, 75, 94, 77, 87, 113, 81, 78, 73, 95, 99, 103, 88, 90, 94, 72, 76, 74, 84, 85, 95, 69, 70, 93, 101, 94, 89, 79, 96, 79, 88, 91, 78, 72, 94, 84, 90, 90, 88, 75, 89, 76, 93, 97, 94, 80, 84, 68, 88, 74, 85, 89, 87, 81, 94, 92, 96, 109, 81, 82, 71, 69, 75, 75, 78, 76, 85, 100, 62, 85, 79, 70, 79, 112, 91, 93, 92, 96, 88, 95, 98, 68, 97, 79, 71, 84, 81, 96, 87, 79, 79, 72, 78, 77, 95, 87, 78, 89, 74, 90, 95, 70, 98, 85, 71, 89, 73, 80, 91, 90, 76, 84, 81, 93, 85, 85, 79, 97, 80, 98, 94, 78, 79, 75, 83] payload_rows_per_dst=[5464, 5272, 5412, 5388, 5542, 5466, 5230, 5326] dup_rows_per_dst=[2766, 2718, 3022, 2812, 2794, 2850, 2706, 2768] combine_rows_per_src=[5464, 5272, 5412, 5388, 5542, 5466, 5230, 5326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43336 payload_rows_remote=37852 dup_rows=22200 dup_rows_remote=19314 dedup_ratio=0.338745 router_tokens_per_expert=[73, 100, 76, 74, 80, 93, 88, 93, 79, 102, 68, 98, 78, 91, 81, 89, 79, 95, 97, 79, 87, 104, 76, 78, 83, 87, 98, 90, 74, 87, 89, 107, 79, 81, 73, 88, 79, 81, 80, 94, 77, 91, 85, 69, 92, 95, 88, 74, 94, 78, 88, 89, 84, 88, 86, 92, 74, 90, 97, 79, 95, 94, 93, 104, 84, 88, 85, 91, 83, 91, 82, 96, 85, 87, 73, 79, 90, 77, 89, 91, 76, 96, 92, 88, 96, 86, 70, 98, 80, 91, 71, 90, 73, 91, 68, 88, 106, 76, 84, 87, 91, 94, 85, 76, 72, 73, 84, 87, 90, 88, 94, 68, 71, 93, 80, 90, 67, 82, 86, 102, 96, 85, 84, 80, 74, 60, 98, 74, 87, 84, 77, 92, 111, 75, 92, 94, 92, 74, 90, 93, 101, 78, 88, 73, 93, 91, 104, 81, 75, 76, 67, 81, 81, 80, 73, 82, 83, 96, 93, 68, 81, 106, 96, 69, 86, 81, 91, 102, 90, 90, 85, 79, 92, 82, 78, 90, 89, 70, 93, 83, 94, 88, 80, 88, 100, 80, 73, 94, 79, 75, 77, 98, 73, 92, 97, 87, 87, 82, 72, 82, 73, 85, 86, 93, 98, 81, 96, 94, 86, 80, 85, 79, 78, 95, 83, 73, 92, 76, 89, 90, 79, 84, 99, 85, 77, 90, 78, 91, 96, 91, 76, 87, 102, 76, 72, 72, 84, 96, 79, 82, 72, 95, 77, 73, 73, 78, 95, 79, 85, 97, 89, 93, 84, 74, 89, 76, 89, 96, 95, 90, 107, 80, 87, 81, 77, 64, 87, 83, 81, 68, 100, 95, 83, 94, 73, 76, 80, 101, 74, 94, 94, 100, 91, 96, 86, 77, 85, 87, 87, 77, 91, 85, 98, 85, 91, 76, 77, 84, 80, 77, 79, 97, 85, 88, 81, 90, 78, 100, 57, 91, 81, 96, 86, 84, 102, 86, 89, 103, 74, 83, 96, 95, 88, 77, 81, 93, 95, 85, 83, 96, 87, 72, 84, 96, 74, 83, 84, 79, 79, 79, 84, 90, 65, 88, 74, 73, 77, 84, 103, 84, 84, 93, 73, 87, 78, 83, 93, 90, 89, 94, 83, 89, 73, 87, 94, 88, 80, 80, 91, 107, 84, 92, 75, 86, 76, 84, 85, 90, 95, 78, 96, 78, 84, 73] payload_rows_per_dst=[5416, 5462, 5374, 5484, 5438, 5386, 5336, 5440] dup_rows_per_dst=[2782, 2838, 2782, 2886, 2722, 2814, 2706, 2670] combine_rows_per_src=[5416, 5462, 5374, 5484, 5438, 5386, 5336, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43296 payload_rows_remote=37818 dup_rows=22240 dup_rows_remote=19390 dedup_ratio=0.339355 router_tokens_per_expert=[104, 98, 84, 85, 81, 84, 87, 103, 83, 97, 105, 111, 67, 72, 71, 100, 89, 83, 92, 77, 70, 90, 61, 87, 95, 72, 81, 87, 85, 97, 91, 98, 91, 88, 71, 75, 89, 88, 76, 69, 91, 100, 84, 89, 81, 86, 84, 88, 73, 77, 91, 89, 83, 93, 97, 83, 88, 92, 100, 77, 68, 91, 93, 100, 84, 79, 77, 84, 74, 110, 76, 90, 76, 92, 90, 97, 93, 94, 85, 89, 76, 77, 76, 86, 90, 75, 96, 94, 76, 78, 75, 100, 65, 89, 92, 86, 102, 81, 68, 87, 75, 92, 91, 84, 95, 78, 78, 101, 77, 96, 89, 73, 91, 68, 83, 88, 101, 83, 86, 84, 79, 92, 92, 89, 83, 92, 83, 80, 109, 86, 88, 76, 90, 84, 78, 92, 95, 62, 78, 78, 84, 79, 80, 85, 67, 92, 88, 94, 97, 64, 80, 92, 80, 79, 83, 77, 73, 94, 75, 98, 98, 85, 94, 78, 86, 93, 87, 78, 105, 85, 75, 92, 84, 77, 82, 91, 96, 83, 78, 78, 77, 90, 100, 87, 87, 78, 92, 76, 78, 84, 89, 79, 91, 88, 98, 85, 84, 67, 88, 92, 90, 97, 94, 89, 95, 82, 101, 88, 78, 89, 89, 84, 90, 69, 88, 80, 80, 89, 79, 76, 85, 81, 75, 72, 102, 85, 97, 73, 87, 76, 102, 103, 88, 90, 89, 99, 89, 83, 84, 84, 76, 79, 97, 65, 90, 69, 100, 100, 98, 79, 82, 80, 85, 92, 86, 69, 70, 100, 91, 91, 73, 77, 72, 83, 68, 69, 89, 72, 85, 78, 80, 96, 92, 84, 83, 89, 84, 94, 74, 87, 73, 76, 86, 75, 85, 96, 91, 86, 78, 79, 87, 103, 79, 90, 76, 82, 75, 69, 65, 86, 89, 91, 91, 86, 95, 73, 91, 88, 74, 92, 87, 92, 85, 83, 99, 72, 92, 77, 95, 85, 72, 89, 87, 105, 84, 96, 85, 94, 77, 69, 70, 87, 101, 80, 85, 72, 90, 88, 92, 101, 95, 83, 99, 86, 90, 87, 71, 97, 69, 85, 84, 77, 72, 71, 86, 77, 83, 73, 91, 99, 92, 98, 65, 85, 96, 83, 68, 82, 73, 92, 111, 104, 86, 80, 76, 83, 97, 81, 66, 105, 83, 101, 97, 86] payload_rows_per_dst=[5424, 5450, 5442, 5386, 5478, 5318, 5358, 5440] dup_rows_per_dst=[2850, 2782, 2728, 2804, 2850, 2674, 2720, 2832] combine_rows_per_src=[5424, 5450, 5442, 5386, 5478, 5318, 5358, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43226 payload_rows_remote=37798 dup_rows=22310 dup_rows_remote=19532 dedup_ratio=0.340424 router_tokens_per_expert=[83, 106, 85, 80, 90, 85, 100, 84, 80, 87, 86, 96, 91, 83, 88, 72, 105, 85, 97, 92, 67, 82, 75, 85, 84, 91, 107, 97, 83, 91, 89, 81, 92, 70, 77, 68, 103, 83, 104, 74, 73, 85, 92, 79, 86, 96, 95, 89, 85, 81, 75, 86, 83, 89, 87, 90, 98, 77, 98, 108, 101, 94, 65, 86, 71, 81, 84, 71, 71, 84, 83, 83, 82, 95, 86, 91, 85, 86, 81, 93, 95, 88, 104, 80, 85, 86, 84, 95, 81, 80, 95, 73, 74, 90, 73, 90, 90, 90, 76, 102, 87, 95, 77, 77, 84, 74, 75, 88, 84, 99, 94, 95, 91, 90, 67, 68, 86, 79, 84, 88, 100, 88, 66, 84, 81, 75, 98, 78, 93, 89, 77, 91, 77, 71, 89, 81, 97, 82, 79, 82, 80, 82, 99, 80, 96, 85, 70, 80, 83, 82, 67, 79, 86, 58, 72, 86, 87, 86, 70, 103, 81, 89, 99, 85, 96, 73, 89, 85, 70, 95, 80, 102, 86, 77, 91, 80, 84, 93, 84, 83, 76, 82, 88, 101, 108, 84, 93, 84, 80, 82, 81, 88, 86, 68, 85, 101, 75, 102, 82, 88, 86, 96, 87, 70, 74, 76, 71, 70, 82, 95, 82, 81, 80, 90, 99, 90, 90, 66, 114, 93, 75, 98, 81, 68, 70, 85, 80, 91, 88, 92, 85, 93, 85, 83, 77, 91, 93, 92, 90, 73, 76, 68, 80, 106, 89, 83, 89, 97, 89, 77, 101, 88, 85, 85, 93, 88, 96, 67, 85, 84, 100, 95, 92, 68, 76, 87, 87, 93, 90, 86, 93, 84, 86, 72, 79, 115, 72, 58, 71, 85, 71, 85, 86, 84, 86, 76, 76, 87, 92, 92, 74, 73, 76, 101, 85, 77, 97, 88, 92, 100, 67, 100, 98, 99, 88, 73, 79, 84, 81, 93, 78, 94, 95, 78, 97, 91, 91, 100, 93, 84, 72, 96, 69, 76, 88, 96, 100, 91, 99, 92, 74, 92, 85, 84, 77, 82, 88, 80, 76, 102, 90, 77, 89, 64, 89, 81, 94, 99, 87, 87, 91, 86, 79, 96, 101, 71, 65, 77, 72, 87, 79, 103, 76, 78, 75, 70, 79, 106, 89, 78, 85, 82, 71, 89, 89, 89, 97, 84, 92, 81, 93, 84, 72, 87] payload_rows_per_dst=[5442, 5428, 5382, 5462, 5354, 5396, 5386, 5376] dup_rows_per_dst=[2904, 2778, 2736, 2850, 2784, 2736, 2786, 2736] combine_rows_per_src=[5442, 5428, 5382, 5462, 5354, 5396, 5386, 5376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43286 payload_rows_remote=37926 dup_rows=22250 dup_rows_remote=19408 dedup_ratio=0.339508 router_tokens_per_expert=[96, 92, 69, 79, 81, 85, 77, 89, 89, 86, 87, 92, 101, 98, 93, 90, 84, 73, 74, 73, 92, 92, 92, 87, 93, 93, 96, 87, 100, 82, 84, 100, 84, 87, 82, 87, 94, 81, 81, 82, 81, 81, 100, 86, 85, 92, 90, 95, 87, 89, 70, 91, 93, 95, 86, 72, 91, 79, 78, 89, 80, 80, 82, 81, 70, 85, 90, 72, 75, 97, 74, 98, 80, 86, 76, 84, 72, 70, 82, 98, 87, 84, 87, 93, 89, 84, 81, 77, 90, 80, 91, 83, 63, 107, 87, 93, 81, 77, 80, 69, 95, 95, 65, 91, 75, 73, 100, 81, 91, 84, 93, 85, 87, 96, 103, 88, 111, 75, 81, 94, 85, 86, 95, 75, 99, 81, 75, 90, 101, 90, 81, 64, 77, 81, 88, 83, 92, 103, 90, 80, 95, 91, 83, 86, 79, 90, 79, 75, 92, 78, 70, 78, 79, 88, 109, 76, 100, 85, 81, 80, 69, 93, 102, 74, 79, 79, 83, 82, 84, 84, 79, 84, 78, 89, 83, 99, 78, 83, 85, 111, 89, 91, 69, 69, 89, 86, 79, 67, 78, 92, 70, 81, 94, 83, 81, 79, 80, 78, 96, 80, 89, 87, 91, 92, 102, 84, 77, 79, 81, 103, 78, 103, 63, 101, 85, 76, 77, 70, 96, 96, 93, 94, 85, 95, 79, 96, 93, 90, 73, 74, 92, 99, 76, 97, 91, 82, 95, 97, 93, 73, 99, 92, 81, 89, 92, 74, 72, 74, 91, 99, 81, 80, 84, 75, 92, 99, 77, 77, 87, 86, 83, 91, 86, 80, 92, 99, 70, 84, 89, 92, 84, 85, 87, 87, 88, 97, 66, 86, 88, 75, 69, 80, 82, 91, 73, 84, 106, 79, 87, 77, 67, 73, 100, 90, 82, 77, 89, 89, 85, 78, 82, 75, 104, 88, 99, 95, 90, 92, 81, 80, 97, 92, 88, 85, 102, 78, 91, 84, 95, 81, 74, 99, 78, 74, 87, 85, 80, 74, 80, 75, 78, 86, 83, 91, 90, 86, 87, 79, 83, 88, 84, 86, 94, 76, 81, 100, 78, 91, 115, 85, 79, 99, 92, 75, 85, 71, 97, 90, 78, 83, 83, 87, 85, 63, 80, 81, 86, 72, 73, 92, 86, 90, 75, 95, 88, 93, 84, 74, 87, 97, 82, 88, 82, 74] payload_rows_per_dst=[5400, 5482, 5372, 5350, 5408, 5428, 5486, 5360] dup_rows_per_dst=[2804, 2758, 2720, 2644, 2730, 2910, 2842, 2842] combine_rows_per_src=[5400, 5482, 5372, 5350, 5408, 5428, 5486, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43136 payload_rows_remote=37690 dup_rows=22400 dup_rows_remote=19454 dedup_ratio=0.341797 router_tokens_per_expert=[81, 89, 88, 88, 85, 94, 85, 92, 86, 75, 83, 97, 80, 87, 90, 94, 80, 66, 77, 93, 71, 92, 75, 81, 78, 92, 82, 91, 89, 70, 76, 88, 102, 78, 80, 81, 79, 89, 74, 101, 82, 86, 74, 67, 85, 72, 69, 83, 83, 96, 80, 84, 79, 85, 95, 89, 71, 76, 84, 99, 84, 79, 85, 72, 77, 90, 74, 80, 90, 90, 75, 58, 87, 80, 81, 101, 81, 83, 79, 74, 71, 65, 91, 81, 95, 83, 88, 83, 92, 80, 109, 88, 82, 78, 95, 92, 106, 72, 92, 90, 96, 99, 101, 85, 98, 101, 88, 96, 73, 96, 74, 94, 88, 91, 70, 90, 86, 77, 81, 89, 67, 85, 107, 84, 79, 94, 77, 96, 83, 73, 76, 92, 95, 92, 82, 91, 87, 87, 71, 90, 90, 84, 93, 88, 91, 87, 83, 72, 79, 60, 99, 75, 82, 77, 87, 86, 76, 83, 86, 96, 90, 100, 77, 69, 87, 87, 85, 109, 81, 80, 89, 87, 87, 99, 94, 77, 68, 92, 92, 83, 82, 101, 83, 81, 107, 79, 72, 87, 88, 88, 98, 86, 80, 83, 94, 91, 91, 93, 84, 77, 77, 91, 93, 84, 91, 83, 88, 80, 102, 81, 84, 86, 87, 81, 87, 105, 77, 92, 81, 85, 88, 92, 95, 72, 74, 75, 89, 82, 97, 94, 96, 88, 75, 75, 85, 67, 96, 88, 96, 67, 70, 85, 91, 79, 83, 84, 96, 82, 82, 84, 82, 82, 76, 75, 81, 96, 86, 78, 85, 88, 79, 89, 85, 95, 81, 72, 72, 83, 91, 84, 80, 67, 89, 85, 78, 77, 84, 84, 97, 81, 93, 83, 91, 79, 83, 72, 77, 87, 67, 90, 73, 96, 86, 91, 99, 86, 93, 88, 71, 92, 92, 89, 89, 89, 78, 91, 87, 101, 84, 98, 83, 89, 71, 71, 81, 102, 76, 94, 93, 108, 85, 98, 88, 83, 97, 85, 82, 86, 106, 94, 88, 82, 99, 93, 96, 92, 97, 85, 106, 85, 79, 88, 78, 84, 75, 92, 72, 83, 84, 95, 89, 70, 87, 72, 99, 77, 93, 90, 66, 96, 112, 108, 69, 86, 89, 86, 98, 82, 104, 103, 82, 85, 76, 82, 72, 99, 84, 71, 65, 84, 91, 90, 71, 72] payload_rows_per_dst=[5240, 5354, 5446, 5498, 5470, 5274, 5462, 5392] dup_rows_per_dst=[2754, 2674, 2946, 2872, 2768, 2692, 2880, 2814] combine_rows_per_src=[5240, 5354, 5446, 5498, 5470, 5274, 5462, 5392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43178 payload_rows_remote=37796 dup_rows=22358 dup_rows_remote=19534 dedup_ratio=0.341156 router_tokens_per_expert=[86, 86, 92, 86, 102, 79, 103, 70, 88, 100, 61, 93, 84, 71, 79, 96, 74, 73, 93, 95, 107, 90, 70, 107, 84, 102, 75, 78, 91, 96, 86, 92, 98, 70, 86, 68, 92, 83, 89, 85, 99, 82, 92, 74, 90, 101, 84, 83, 90, 92, 101, 74, 85, 92, 82, 84, 93, 98, 77, 91, 68, 83, 74, 98, 76, 90, 88, 73, 76, 83, 76, 75, 71, 80, 84, 100, 84, 86, 93, 100, 68, 99, 88, 97, 71, 70, 71, 76, 79, 85, 96, 92, 89, 82, 93, 79, 77, 82, 79, 98, 92, 101, 82, 86, 79, 95, 94, 85, 88, 86, 92, 90, 72, 82, 72, 95, 80, 90, 93, 81, 76, 71, 77, 86, 100, 83, 80, 82, 78, 87, 90, 97, 88, 67, 89, 83, 102, 99, 83, 96, 90, 83, 84, 79, 82, 85, 95, 70, 98, 81, 75, 70, 81, 74, 81, 71, 101, 83, 91, 84, 89, 102, 75, 83, 82, 82, 80, 93, 80, 86, 77, 79, 103, 87, 84, 84, 90, 86, 85, 93, 76, 70, 78, 80, 112, 93, 87, 97, 94, 87, 82, 85, 80, 69, 79, 87, 96, 91, 90, 68, 78, 78, 94, 81, 85, 78, 100, 83, 80, 85, 101, 90, 73, 82, 115, 96, 70, 98, 90, 86, 92, 98, 75, 93, 75, 66, 83, 84, 60, 68, 73, 88, 82, 88, 81, 69, 99, 86, 85, 88, 82, 92, 71, 79, 89, 85, 83, 93, 75, 93, 77, 77, 78, 72, 80, 75, 85, 100, 92, 87, 90, 71, 93, 87, 82, 81, 87, 82, 79, 81, 91, 71, 67, 94, 83, 83, 98, 86, 77, 102, 99, 86, 70, 89, 78, 77, 71, 98, 80, 93, 103, 78, 91, 98, 97, 76, 89, 82, 87, 72, 77, 78, 74, 105, 87, 82, 99, 75, 85, 84, 86, 102, 72, 101, 90, 89, 90, 82, 96, 83, 78, 78, 92, 93, 94, 85, 76, 69, 88, 87, 89, 80, 80, 97, 64, 63, 72, 95, 80, 85, 83, 91, 75, 78, 77, 86, 91, 91, 82, 91, 99, 91, 73, 91, 85, 84, 93, 74, 94, 89, 97, 92, 89, 94, 78, 97, 81, 94, 79, 88, 86, 85, 89, 104, 84, 95, 102, 96, 87, 94, 85, 83, 85, 83] payload_rows_per_dst=[5406, 5382, 5346, 5356, 5368, 5382, 5410, 5528] dup_rows_per_dst=[2872, 2774, 2726, 2810, 2704, 2824, 2782, 2866] combine_rows_per_src=[5406, 5382, 5346, 5356, 5368, 5382, 5410, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43486 payload_rows_remote=37904 dup_rows=22050 dup_rows_remote=19186 dedup_ratio=0.336456 router_tokens_per_expert=[83, 98, 91, 78, 85, 82, 93, 72, 83, 89, 72, 89, 78, 74, 76, 73, 96, 85, 75, 88, 92, 87, 70, 97, 68, 71, 105, 91, 83, 94, 94, 78, 71, 86, 97, 85, 100, 79, 87, 84, 98, 104, 85, 72, 85, 115, 86, 97, 72, 94, 78, 73, 77, 102, 82, 73, 89, 87, 93, 94, 80, 83, 108, 80, 95, 96, 77, 83, 89, 103, 86, 82, 66, 95, 104, 87, 93, 83, 73, 73, 88, 67, 87, 88, 58, 93, 99, 90, 90, 88, 93, 68, 75, 81, 81, 81, 69, 85, 94, 94, 106, 85, 97, 81, 78, 82, 100, 107, 88, 84, 94, 90, 91, 92, 82, 82, 83, 96, 65, 95, 91, 69, 78, 79, 90, 98, 85, 82, 89, 79, 82, 105, 73, 84, 78, 74, 92, 89, 84, 96, 78, 69, 87, 85, 76, 75, 94, 84, 70, 72, 89, 75, 61, 85, 61, 67, 95, 96, 86, 87, 76, 83, 92, 88, 88, 76, 96, 84, 93, 79, 88, 91, 102, 85, 72, 94, 81, 80, 90, 78, 87, 73, 67, 83, 79, 87, 82, 95, 102, 73, 82, 68, 82, 88, 81, 92, 85, 94, 91, 63, 69, 67, 85, 67, 87, 88, 80, 99, 78, 91, 86, 84, 83, 70, 88, 102, 83, 81, 82, 79, 94, 96, 101, 74, 90, 77, 85, 81, 75, 98, 74, 107, 91, 83, 90, 95, 79, 76, 96, 76, 97, 83, 75, 81, 91, 81, 79, 92, 91, 81, 79, 89, 93, 105, 95, 93, 86, 83, 86, 94, 89, 66, 75, 91, 94, 97, 85, 85, 64, 71, 80, 96, 72, 102, 92, 88, 84, 93, 74, 75, 93, 71, 80, 98, 74, 84, 90, 91, 89, 84, 77, 80, 97, 96, 85, 92, 78, 84, 96, 85, 82, 89, 117, 81, 105, 79, 85, 93, 73, 102, 94, 76, 91, 86, 104, 94, 94, 98, 97, 80, 96, 107, 99, 78, 95, 83, 97, 84, 69, 81, 78, 69, 81, 84, 77, 82, 88, 89, 104, 87, 65, 104, 94, 81, 79, 84, 87, 97, 83, 94, 87, 93, 71, 88, 78, 71, 77, 88, 81, 75, 83, 77, 85, 91, 64, 97, 89, 83, 66, 80, 76, 90, 76, 97, 94, 90, 94, 81, 81, 96, 91, 81, 95, 71] payload_rows_per_dst=[5346, 5544, 5340, 5330, 5392, 5530, 5582, 5422] dup_rows_per_dst=[2666, 2840, 2762, 2604, 2734, 2856, 2864, 2724] combine_rows_per_src=[5346, 5544, 5340, 5330, 5392, 5530, 5582, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=539549696 dispatch_meta_bytes=229224 combine_payload_bytes=539549696 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=539778920 ingress_bytes=891871232 link_bytes=891871232 token_bytes=1079328616 weight_movement_bytes=352321536 total_nvlink_bytes=1431650152 weight_to_token_ratio=0.326427 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.491159921e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542646272 dispatch_meta_bytes=228664 combine_payload_bytes=542646272 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542874936 ingress_bytes=718807040 link_bytes=718807040 token_bytes=1085521208 weight_movement_bytes=176160768 total_nvlink_bytes=1261681976 weight_to_token_ratio=0.162282 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.037529069e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542158848 dispatch_meta_bytes=228832 combine_payload_bytes=542158848 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=542387680 ingress_bytes=1246801920 link_bytes=1246801920 token_bytes=1084546528 weight_movement_bytes=704643072 total_nvlink_bytes=1789189600 weight_to_token_ratio=0.649712 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.190341 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047235547e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541872128 dispatch_meta_bytes=229320 combine_payload_bytes=541872128 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542101448 ingress_bytes=718032896 link_bytes=718032896 token_bytes=1083973576 weight_movement_bytes=176160768 total_nvlink_bytes=1260134344 weight_to_token_ratio=0.162514 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543707136 dispatch_meta_bytes=229336 combine_payload_bytes=543707136 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543936472 ingress_bytes=719867904 link_bytes=719867904 token_bytes=1087643608 weight_movement_bytes=176160768 total_nvlink_bytes=1263804376 weight_to_token_ratio=0.161966 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.046439662e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1079328616 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=539778920 ingress_bytes=891871232 link_bytes=891871232 total_nvlink_bytes=1431650152 weight_to_token_ratio=0.326427 prefetch_share_of_ingress=0.197518 dedup_ratio=0.342346 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.491159921e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540323840 dispatch_meta_bytes=228576 combine_payload_bytes=540323840 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=540552416 ingress_bytes=892645376 link_bytes=892645376 token_bytes=1080876256 weight_movement_bytes=352321536 total_nvlink_bytes=1433197792 weight_to_token_ratio=0.325959 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.497662246e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084546528 prefetch_bytes=0 reduce_bytes=704643072 weight_movement_bytes=704643072 egress_bytes=542387680 ingress_bytes=1246801920 link_bytes=1246801920 total_nvlink_bytes=1789189600 weight_to_token_ratio=0.649712 prefetch_share_of_ingress=0.000000 dedup_ratio=0.339355 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=4 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047235547e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085521208 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542874936 ingress_bytes=718807040 link_bytes=718807040 total_nvlink_bytes=1261681976 weight_to_token_ratio=0.162282 prefetch_share_of_ingress=0.245074 dedup_ratio=0.338745 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.037529069e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083973576 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542101448 ingress_bytes=718032896 link_bytes=718032896 total_nvlink_bytes=1260134344 weight_to_token_ratio=0.162514 prefetch_share_of_ingress=0.245338 dedup_ratio=0.340424 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541843456 dispatch_meta_bytes=229320 combine_payload_bytes=541843456 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542072776 ingress_bytes=718004224 link_bytes=718004224 token_bytes=1083916232 weight_movement_bytes=176160768 total_nvlink_bytes=1260077000 weight_to_token_ratio=0.162522 gemm_rows_defined=36992 gemm_rows_charged=45056 gemm_tail_pad_rows=8064 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.178977 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.030785917e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543391744 dispatch_meta_bytes=228360 combine_payload_bytes=543391744 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=543620104 ingress_bytes=895713280 link_bytes=895713280 token_bytes=1087011848 weight_movement_bytes=352321536 total_nvlink_bytes=1439333384 weight_to_token_ratio=0.324119 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.523430718e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087643608 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543936472 ingress_bytes=719867904 link_bytes=719867904 total_nvlink_bytes=1263804376 weight_to_token_ratio=0.161966 prefetch_share_of_ingress=0.244713 dedup_ratio=0.339508 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.046439662e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1080876256 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=540552416 ingress_bytes=892645376 link_bytes=892645376 total_nvlink_bytes=1433197792 weight_to_token_ratio=0.325959 prefetch_share_of_ingress=0.197347 dedup_ratio=0.341797 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.497662246e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083916232 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542072776 ingress_bytes=718004224 link_bytes=718004224 total_nvlink_bytes=1260077000 weight_to_token_ratio=0.162522 prefetch_share_of_ingress=0.245348 dedup_ratio=0.341156 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.178977 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.030785917e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087011848 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=543620104 ingress_bytes=895713280 link_bytes=895713280 total_nvlink_bytes=1439333384 weight_to_token_ratio=0.324119 prefetch_share_of_ingress=0.196671 dedup_ratio=0.336456 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.523430718e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 121.2 | throughput per GPU (TFLOP/s/GPU): 292.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.480609E+01 | loss scale: 1.0 | grad norm: 83.279 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43324 payload_rows_remote=37876 dup_rows=22212 dup_rows_remote=19442 dedup_ratio=0.338928 router_tokens_per_expert=[103, 75, 89, 88, 90, 78, 94, 77, 95, 91, 80, 83, 76, 75, 93, 90, 85, 92, 82, 99, 87, 89, 87, 85, 81, 70, 82, 92, 88, 76, 83, 91, 68, 82, 81, 94, 84, 95, 84, 70, 96, 90, 105, 82, 73, 84, 74, 68, 84, 87, 76, 103, 95, 86, 82, 89, 85, 89, 75, 92, 94, 81, 88, 76, 105, 74, 84, 115, 80, 78, 78, 80, 79, 80, 71, 83, 84, 87, 91, 85, 84, 89, 79, 86, 78, 95, 107, 79, 106, 77, 75, 91, 79, 82, 72, 86, 98, 77, 77, 98, 94, 87, 91, 81, 73, 106, 83, 87, 80, 73, 112, 87, 86, 87, 89, 104, 58, 91, 83, 76, 81, 74, 108, 68, 95, 87, 86, 91, 85, 77, 92, 89, 89, 88, 87, 90, 100, 67, 74, 80, 82, 82, 80, 97, 61, 67, 75, 91, 68, 74, 91, 79, 74, 63, 86, 87, 85, 95, 95, 85, 105, 85, 82, 85, 86, 111, 82, 76, 106, 90, 79, 90, 72, 97, 95, 82, 68, 78, 79, 88, 90, 94, 70, 105, 89, 69, 96, 86, 100, 74, 72, 92, 81, 96, 71, 84, 94, 70, 83, 79, 92, 96, 84, 88, 81, 94, 80, 82, 69, 67, 70, 74, 82, 88, 100, 80, 98, 99, 85, 89, 71, 91, 80, 88, 100, 86, 92, 85, 105, 76, 99, 74, 105, 104, 98, 83, 94, 80, 75, 89, 96, 82, 85, 85, 65, 90, 103, 88, 98, 75, 74, 89, 70, 98, 79, 89, 93, 83, 88, 97, 92, 96, 83, 82, 80, 76, 73, 97, 66, 78, 94, 93, 89, 70, 93, 85, 99, 79, 86, 97, 83, 91, 84, 80, 87, 91, 78, 72, 87, 77, 79, 78, 93, 96, 75, 106, 84, 65, 76, 83, 87, 86, 84, 92, 81, 84, 79, 97, 83, 79, 84, 91, 91, 75, 90, 87, 93, 82, 89, 86, 93, 87, 64, 72, 107, 83, 92, 91, 87, 90, 101, 88, 98, 77, 82, 99, 71, 91, 81, 80, 86, 85, 88, 73, 83, 83, 92, 81, 74, 71, 81, 62, 82, 85, 81, 89, 82, 97, 100, 75, 84, 103, 96, 87, 76, 95, 82, 99, 85, 75, 104, 84, 84, 84, 83, 84, 84, 81, 87, 76, 91, 104, 85, 67] payload_rows_per_dst=[5356, 5356, 5354, 5422, 5386, 5492, 5510, 5448] dup_rows_per_dst=[2676, 2644, 2800, 2676, 2948, 2818, 2880, 2770] combine_rows_per_src=[5356, 5356, 5354, 5422, 5386, 5492, 5510, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43198 payload_rows_remote=37810 dup_rows=22338 dup_rows_remote=19610 dedup_ratio=0.340851 router_tokens_per_expert=[65, 97, 92, 72, 75, 96, 90, 97, 99, 100, 86, 78, 90, 89, 82, 87, 86, 81, 80, 93, 81, 88, 59, 78, 97, 92, 82, 78, 100, 78, 69, 85, 82, 82, 91, 78, 89, 79, 92, 85, 98, 77, 91, 80, 74, 69, 74, 75, 100, 91, 85, 84, 87, 86, 85, 77, 89, 82, 85, 85, 98, 75, 82, 71, 87, 87, 95, 79, 78, 84, 85, 78, 77, 94, 92, 89, 82, 76, 77, 80, 72, 68, 88, 88, 103, 97, 85, 87, 102, 78, 93, 86, 75, 94, 97, 88, 73, 77, 78, 74, 89, 64, 88, 86, 82, 99, 85, 85, 87, 89, 81, 97, 94, 93, 92, 103, 80, 95, 94, 71, 103, 83, 85, 102, 86, 79, 84, 82, 88, 82, 78, 80, 90, 98, 89, 96, 94, 81, 73, 70, 86, 89, 87, 85, 90, 103, 92, 85, 86, 74, 94, 84, 86, 91, 80, 77, 79, 88, 80, 73, 97, 86, 76, 86, 96, 97, 85, 79, 83, 83, 75, 85, 81, 76, 88, 78, 104, 67, 83, 97, 97, 82, 88, 71, 77, 89, 85, 86, 88, 76, 71, 84, 75, 73, 66, 104, 86, 83, 79, 95, 85, 74, 83, 81, 89, 95, 91, 89, 95, 76, 95, 90, 88, 82, 100, 111, 95, 91, 92, 94, 54, 76, 83, 95, 91, 81, 63, 97, 93, 68, 84, 87, 71, 89, 104, 78, 87, 97, 99, 83, 86, 87, 73, 88, 87, 94, 63, 80, 83, 88, 97, 80, 97, 94, 82, 113, 83, 100, 83, 93, 87, 74, 80, 79, 68, 95, 115, 91, 78, 69, 91, 81, 78, 85, 90, 86, 83, 83, 83, 89, 85, 82, 80, 71, 83, 71, 80, 75, 87, 101, 78, 80, 84, 94, 85, 100, 84, 88, 82, 84, 78, 71, 100, 77, 87, 86, 93, 77, 86, 84, 78, 94, 83, 86, 96, 101, 88, 82, 100, 89, 81, 69, 89, 83, 80, 78, 99, 96, 88, 96, 92, 83, 81, 91, 84, 75, 77, 80, 74, 91, 86, 104, 90, 79, 76, 87, 99, 76, 75, 83, 92, 75, 101, 87, 91, 65, 82, 94, 93, 102, 91, 66, 105, 82, 88, 90, 87, 84, 81, 69, 79, 62, 92, 80, 86, 102, 80, 108, 83, 83, 71, 83, 85, 99] payload_rows_per_dst=[5356, 5414, 5406, 5388, 5380, 5366, 5484, 5404] dup_rows_per_dst=[2720, 2792, 2846, 2728, 2894, 2760, 2812, 2786] combine_rows_per_src=[5356, 5414, 5406, 5388, 5380, 5366, 5484, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43160 payload_rows_remote=37798 dup_rows=22376 dup_rows_remote=19550 dedup_ratio=0.341431 router_tokens_per_expert=[101, 78, 104, 84, 72, 80, 83, 95, 99, 81, 77, 95, 96, 85, 86, 82, 79, 90, 91, 84, 97, 92, 75, 94, 75, 99, 90, 101, 85, 96, 88, 89, 73, 74, 94, 93, 89, 95, 96, 85, 88, 85, 91, 96, 89, 73, 89, 72, 87, 72, 100, 91, 78, 65, 95, 85, 68, 82, 94, 86, 79, 81, 84, 92, 106, 86, 80, 75, 84, 83, 95, 68, 74, 83, 84, 86, 80, 87, 71, 83, 89, 70, 82, 98, 90, 98, 102, 74, 114, 78, 84, 79, 99, 84, 87, 97, 98, 83, 87, 78, 70, 102, 74, 89, 86, 86, 105, 90, 87, 93, 94, 93, 86, 84, 90, 87, 89, 86, 82, 70, 79, 82, 90, 91, 73, 88, 89, 89, 86, 85, 89, 92, 61, 89, 90, 82, 79, 85, 88, 89, 68, 75, 76, 92, 82, 76, 91, 73, 81, 72, 98, 77, 66, 77, 89, 78, 84, 63, 95, 82, 90, 89, 86, 100, 78, 105, 89, 91, 87, 77, 85, 89, 104, 86, 101, 85, 88, 70, 106, 81, 90, 95, 82, 77, 87, 82, 85, 75, 85, 74, 93, 92, 84, 81, 79, 83, 66, 72, 90, 83, 69, 86, 74, 89, 75, 87, 83, 73, 92, 95, 68, 82, 84, 94, 82, 94, 92, 96, 90, 75, 95, 98, 85, 79, 70, 84, 77, 76, 92, 71, 87, 78, 88, 84, 79, 76, 84, 66, 88, 90, 88, 65, 80, 87, 87, 86, 91, 77, 103, 98, 82, 78, 85, 73, 71, 69, 84, 95, 82, 90, 94, 81, 86, 100, 90, 97, 99, 88, 90, 74, 71, 81, 83, 63, 101, 83, 66, 86, 89, 103, 84, 86, 88, 92, 92, 84, 69, 78, 79, 69, 84, 82, 76, 87, 88, 89, 69, 101, 91, 89, 83, 80, 96, 86, 89, 94, 90, 76, 89, 104, 78, 94, 68, 88, 83, 82, 76, 77, 64, 81, 88, 80, 87, 97, 90, 85, 86, 79, 84, 87, 86, 97, 103, 71, 94, 98, 63, 85, 95, 77, 99, 80, 81, 86, 87, 85, 105, 94, 75, 93, 77, 83, 68, 87, 92, 80, 96, 83, 78, 83, 78, 103, 96, 82, 88, 78, 70, 68, 85, 89, 93, 88, 105, 99, 107, 102, 84, 86, 85, 83, 92, 94, 99, 76] payload_rows_per_dst=[5514, 5400, 5376, 5348, 5294, 5382, 5362, 5484] dup_rows_per_dst=[2896, 2778, 2816, 2828, 2636, 2756, 2826, 2840] combine_rows_per_src=[5514, 5400, 5376, 5348, 5294, 5382, 5362, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43390 payload_rows_remote=38030 dup_rows=22146 dup_rows_remote=19408 dedup_ratio=0.337921 router_tokens_per_expert=[86, 63, 75, 88, 73, 87, 99, 89, 86, 79, 90, 88, 80, 78, 68, 80, 82, 97, 87, 89, 106, 83, 74, 93, 90, 67, 89, 81, 90, 87, 87, 103, 84, 82, 92, 94, 89, 74, 95, 80, 64, 73, 81, 88, 91, 89, 93, 64, 73, 79, 86, 89, 77, 83, 88, 84, 84, 96, 88, 76, 82, 89, 96, 86, 110, 67, 94, 100, 84, 91, 97, 89, 82, 81, 72, 83, 84, 92, 65, 84, 104, 93, 102, 101, 97, 95, 86, 78, 73, 81, 86, 87, 74, 79, 81, 95, 84, 79, 83, 88, 79, 76, 98, 76, 85, 80, 77, 92, 84, 92, 78, 86, 96, 85, 72, 79, 68, 76, 81, 91, 95, 91, 85, 93, 74, 102, 78, 76, 94, 82, 83, 83, 83, 94, 93, 80, 83, 85, 84, 89, 85, 98, 74, 80, 87, 84, 96, 86, 94, 73, 95, 75, 78, 86, 74, 85, 90, 74, 87, 90, 88, 80, 86, 87, 89, 89, 81, 91, 76, 88, 87, 74, 88, 88, 91, 91, 88, 90, 84, 82, 85, 64, 76, 82, 83, 72, 93, 80, 91, 83, 100, 96, 76, 91, 92, 80, 87, 88, 75, 82, 83, 84, 100, 88, 97, 91, 89, 74, 74, 98, 81, 89, 102, 95, 77, 82, 73, 96, 74, 85, 86, 76, 81, 65, 70, 81, 88, 92, 76, 74, 84, 81, 89, 80, 71, 87, 93, 75, 72, 81, 80, 73, 83, 80, 80, 72, 89, 77, 80, 89, 70, 92, 85, 87, 76, 87, 82, 94, 95, 85, 84, 76, 88, 72, 89, 88, 100, 90, 92, 97, 67, 84, 90, 86, 107, 84, 97, 95, 80, 77, 84, 84, 83, 77, 78, 78, 91, 108, 95, 98, 86, 93, 84, 92, 77, 71, 85, 91, 75, 79, 79, 88, 89, 85, 81, 91, 91, 90, 71, 72, 99, 97, 91, 96, 73, 72, 94, 84, 100, 73, 81, 99, 94, 110, 83, 109, 80, 86, 103, 88, 86, 81, 93, 97, 69, 90, 79, 91, 87, 84, 89, 82, 110, 110, 103, 70, 81, 85, 83, 96, 67, 71, 93, 78, 76, 103, 92, 84, 77, 81, 96, 87, 84, 106, 64, 93, 84, 76, 90, 84, 84, 93, 85, 96, 88, 105, 96, 87, 92, 73, 100, 80, 87, 72] payload_rows_per_dst=[5358, 5448, 5360, 5466, 5326, 5470, 5494, 5468] dup_rows_per_dst=[2736, 2838, 2738, 2760, 2684, 2694, 2888, 2808] combine_rows_per_src=[5358, 5448, 5360, 5466, 5326, 5470, 5494, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43198 payload_rows_remote=37828 dup_rows=22338 dup_rows_remote=19636 dedup_ratio=0.340851 router_tokens_per_expert=[93, 102, 89, 88, 74, 101, 65, 89, 74, 84, 72, 86, 129, 87, 104, 68, 96, 79, 87, 96, 71, 84, 74, 79, 83, 91, 95, 97, 105, 86, 80, 100, 97, 106, 85, 97, 87, 69, 100, 89, 76, 79, 93, 78, 103, 98, 89, 92, 75, 84, 80, 89, 91, 76, 101, 85, 71, 83, 104, 79, 97, 101, 78, 78, 74, 91, 94, 77, 80, 75, 87, 69, 87, 63, 71, 83, 87, 74, 93, 96, 86, 85, 74, 85, 94, 96, 90, 80, 97, 84, 86, 81, 85, 90, 81, 69, 92, 86, 94, 81, 69, 94, 89, 80, 80, 79, 76, 88, 93, 83, 83, 94, 84, 75, 76, 90, 102, 84, 85, 65, 81, 102, 68, 81, 103, 64, 93, 74, 97, 91, 89, 83, 91, 75, 94, 91, 92, 83, 73, 77, 76, 81, 89, 79, 86, 76, 95, 97, 74, 79, 102, 89, 84, 83, 84, 81, 74, 75, 99, 80, 75, 86, 76, 79, 88, 62, 93, 78, 89, 90, 92, 87, 76, 81, 66, 90, 81, 66, 95, 77, 85, 81, 82, 85, 97, 102, 89, 71, 88, 88, 89, 90, 86, 85, 72, 84, 103, 84, 92, 79, 88, 109, 89, 99, 81, 101, 85, 84, 91, 97, 93, 108, 83, 81, 84, 97, 91, 82, 75, 88, 89, 91, 93, 106, 83, 79, 80, 89, 86, 97, 85, 96, 95, 77, 75, 86, 86, 69, 84, 87, 91, 72, 93, 92, 92, 88, 74, 72, 86, 69, 82, 97, 88, 77, 108, 85, 73, 89, 92, 77, 97, 81, 59, 81, 86, 96, 81, 89, 79, 105, 92, 106, 98, 91, 90, 87, 77, 87, 68, 87, 102, 77, 93, 85, 94, 91, 96, 82, 69, 93, 89, 90, 77, 88, 94, 92, 84, 100, 81, 96, 103, 73, 72, 76, 79, 75, 70, 81, 79, 83, 85, 97, 92, 90, 84, 101, 58, 85, 88, 86, 71, 85, 80, 83, 72, 77, 90, 102, 77, 76, 63, 70, 94, 91, 72, 76, 75, 91, 79, 76, 101, 103, 72, 76, 86, 92, 68, 82, 85, 86, 74, 77, 87, 72, 76, 77, 84, 84, 91, 93, 88, 81, 86, 96, 93, 85, 94, 93, 84, 99, 86, 79, 102, 70, 80, 87, 67, 90, 87, 79, 67, 86, 85, 87] payload_rows_per_dst=[5454, 5370, 5386, 5384, 5514, 5506, 5340, 5244] dup_rows_per_dst=[3038, 2702, 2712, 2884, 2934, 2802, 2638, 2628] combine_rows_per_src=[5454, 5370, 5386, 5384, 5514, 5506, 5340, 5244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43280 payload_rows_remote=37836 dup_rows=22256 dup_rows_remote=19488 dedup_ratio=0.339600 router_tokens_per_expert=[72, 82, 90, 89, 77, 82, 93, 91, 87, 93, 74, 103, 86, 98, 68, 97, 92, 81, 85, 89, 83, 73, 91, 69, 77, 69, 95, 87, 83, 77, 95, 89, 80, 71, 89, 84, 91, 71, 70, 92, 76, 82, 85, 72, 89, 87, 90, 91, 89, 103, 101, 80, 77, 85, 80, 83, 110, 80, 75, 93, 88, 87, 115, 84, 76, 94, 76, 82, 84, 84, 85, 85, 96, 82, 81, 115, 92, 84, 81, 86, 100, 95, 87, 94, 80, 88, 84, 84, 86, 70, 77, 82, 74, 97, 97, 95, 95, 73, 91, 95, 71, 82, 86, 101, 86, 68, 78, 84, 83, 99, 92, 67, 86, 70, 81, 95, 76, 92, 87, 94, 87, 101, 78, 86, 78, 84, 99, 72, 74, 93, 92, 69, 86, 87, 83, 94, 73, 70, 77, 78, 89, 96, 94, 82, 95, 92, 87, 80, 74, 84, 88, 71, 84, 82, 82, 95, 80, 87, 81, 94, 94, 91, 90, 91, 84, 87, 80, 90, 76, 78, 93, 94, 83, 87, 92, 85, 89, 84, 95, 79, 83, 78, 81, 70, 89, 90, 87, 88, 90, 84, 76, 77, 79, 103, 91, 76, 79, 105, 88, 73, 93, 77, 73, 107, 84, 92, 88, 95, 76, 68, 81, 95, 88, 86, 92, 106, 84, 81, 65, 99, 92, 84, 80, 69, 92, 79, 88, 76, 84, 80, 76, 93, 96, 92, 73, 78, 89, 72, 98, 90, 85, 73, 86, 98, 85, 93, 68, 92, 97, 79, 86, 103, 83, 106, 72, 59, 95, 98, 87, 77, 83, 83, 91, 79, 99, 85, 98, 78, 90, 92, 75, 100, 77, 83, 100, 77, 72, 86, 109, 69, 84, 79, 78, 92, 85, 74, 93, 73, 77, 90, 81, 84, 92, 89, 69, 80, 78, 73, 84, 85, 84, 68, 77, 86, 87, 72, 95, 80, 104, 84, 86, 88, 92, 85, 93, 67, 97, 83, 97, 95, 90, 95, 92, 71, 73, 86, 89, 82, 69, 94, 92, 85, 83, 86, 88, 69, 95, 83, 86, 83, 84, 91, 94, 78, 89, 92, 99, 71, 88, 80, 86, 77, 88, 106, 80, 80, 77, 89, 86, 87, 84, 100, 93, 98, 91, 89, 99, 85, 71, 89, 68, 80, 91, 83, 91, 91, 76, 82, 75, 82, 82, 80, 92, 85] payload_rows_per_dst=[5358, 5496, 5398, 5358, 5356, 5444, 5430, 5440] dup_rows_per_dst=[2716, 2910, 2710, 2824, 2854, 2768, 2662, 2812] combine_rows_per_src=[5358, 5496, 5398, 5358, 5356, 5444, 5430, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43268 payload_rows_remote=37886 dup_rows=22268 dup_rows_remote=19502 dedup_ratio=0.339783 router_tokens_per_expert=[97, 79, 89, 97, 85, 90, 79, 101, 88, 94, 98, 69, 79, 95, 93, 81, 84, 83, 78, 87, 79, 90, 85, 90, 84, 82, 94, 77, 87, 90, 99, 101, 83, 80, 86, 83, 57, 78, 68, 82, 76, 70, 76, 75, 82, 94, 96, 84, 88, 94, 76, 96, 83, 82, 89, 99, 96, 98, 85, 92, 98, 80, 100, 73, 98, 89, 86, 77, 76, 82, 100, 85, 83, 72, 88, 84, 78, 100, 82, 78, 78, 88, 91, 94, 97, 86, 79, 73, 79, 86, 88, 83, 70, 93, 87, 74, 82, 82, 88, 88, 94, 71, 93, 90, 98, 99, 78, 108, 83, 88, 80, 86, 86, 75, 86, 73, 87, 88, 111, 80, 92, 93, 85, 97, 91, 92, 73, 94, 105, 96, 104, 81, 91, 84, 96, 70, 76, 89, 86, 84, 91, 79, 72, 101, 97, 83, 92, 93, 78, 80, 90, 87, 83, 94, 81, 76, 82, 78, 95, 84, 79, 80, 91, 84, 77, 94, 89, 71, 80, 61, 80, 91, 91, 94, 81, 87, 87, 86, 87, 112, 96, 82, 79, 85, 89, 83, 91, 82, 67, 77, 83, 90, 75, 67, 80, 78, 70, 98, 86, 102, 96, 75, 73, 83, 75, 101, 86, 105, 80, 102, 81, 86, 74, 72, 94, 82, 88, 84, 98, 73, 83, 95, 81, 68, 82, 92, 67, 76, 82, 74, 78, 94, 90, 98, 96, 80, 97, 86, 79, 77, 96, 76, 82, 90, 87, 91, 90, 59, 81, 88, 82, 102, 75, 96, 78, 89, 84, 93, 84, 92, 85, 90, 88, 74, 96, 67, 97, 74, 79, 75, 84, 88, 84, 100, 79, 93, 68, 84, 75, 63, 86, 71, 93, 86, 96, 84, 99, 84, 93, 89, 79, 88, 92, 84, 84, 80, 80, 77, 86, 84, 72, 93, 87, 92, 80, 90, 94, 74, 86, 92, 71, 70, 90, 100, 89, 83, 98, 85, 89, 85, 94, 90, 80, 75, 69, 88, 75, 74, 77, 74, 100, 96, 83, 84, 92, 86, 93, 87, 105, 98, 81, 84, 88, 85, 93, 87, 87, 95, 77, 92, 71, 73, 87, 95, 89, 91, 76, 100, 72, 83, 83, 85, 62, 77, 98, 105, 82, 75, 73, 70, 87, 74, 93, 90, 98, 80, 92, 99, 79, 76, 64, 91, 86, 89] payload_rows_per_dst=[5382, 5414, 5558, 5486, 5360, 5370, 5378, 5320] dup_rows_per_dst=[2766, 2878, 2848, 2858, 2718, 2744, 2768, 2688] combine_rows_per_src=[5382, 5414, 5558, 5486, 5360, 5370, 5378, 5320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542990336 dispatch_meta_bytes=229272 combine_payload_bytes=542990336 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=543219608 ingress_bytes=895311872 link_bytes=895311872 token_bytes=1086209944 weight_movement_bytes=352321536 total_nvlink_bytes=1438531480 weight_to_token_ratio=0.324359 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.520059142e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542044160 dispatch_meta_bytes=229680 combine_payload_bytes=542044160 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542273840 ingress_bytes=718204928 link_bytes=718204928 token_bytes=1084318000 weight_movement_bytes=176160768 total_nvlink_bytes=1260478768 weight_to_token_ratio=0.162462 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.032471705e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541872128 dispatch_meta_bytes=229392 combine_payload_bytes=541872128 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542101520 ingress_bytes=718032896 link_bytes=718032896 token_bytes=1083973648 weight_movement_bytes=176160768 total_nvlink_bytes=1260134416 weight_to_token_ratio=0.162514 gemm_rows_defined=36864 gemm_rows_charged=45056 gemm_tail_pad_rows=8192 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.181818 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=545198080 dispatch_meta_bytes=229752 combine_payload_bytes=545198080 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=545427832 ingress_bytes=1073680384 link_bytes=1073680384 token_bytes=1090625912 weight_movement_bytes=528482304 total_nvlink_bytes=1619108216 weight_to_token_ratio=0.484568 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.018242961e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542302208 dispatch_meta_bytes=229856 combine_payload_bytes=542302208 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=542532064 ingress_bytes=1070784512 link_bytes=1070784512 token_bytes=1084834272 weight_movement_bytes=528482304 total_nvlink_bytes=1613316576 weight_to_token_ratio=0.487155 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.993919450e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086209944 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=543219608 ingress_bytes=895311872 link_bytes=895311872 total_nvlink_bytes=1438531480 weight_to_token_ratio=0.324359 prefetch_share_of_ingress=0.196759 dedup_ratio=0.338928 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.520059142e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542416896 dispatch_meta_bytes=229296 combine_payload_bytes=542416896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542646192 ingress_bytes=718577664 link_bytes=718577664 token_bytes=1085063088 weight_movement_bytes=176160768 total_nvlink_bytes=1261223856 weight_to_token_ratio=0.162351 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084318000 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542273840 ingress_bytes=718204928 link_bytes=718204928 total_nvlink_bytes=1260478768 weight_to_token_ratio=0.162462 prefetch_share_of_ingress=0.245279 dedup_ratio=0.340851 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.032471705e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083973648 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542101520 ingress_bytes=718032896 link_bytes=718032896 total_nvlink_bytes=1260134416 weight_to_token_ratio=0.162514 prefetch_share_of_ingress=0.245338 dedup_ratio=0.341431 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.181818 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031026744e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543133696 dispatch_meta_bytes=229552 combine_payload_bytes=543133696 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=543363248 ingress_bytes=895455232 link_bytes=895455232 token_bytes=1086496944 weight_movement_bytes=352321536 total_nvlink_bytes=1438818480 weight_to_token_ratio=0.324273 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.190341 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.521263277e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1090625912 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=545427832 ingress_bytes=1073680384 link_bytes=1073680384 total_nvlink_bytes=1619108216 weight_to_token_ratio=0.484568 prefetch_share_of_ingress=0.164072 dedup_ratio=0.337921 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.018242961e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084834272 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=542532064 ingress_bytes=1070784512 link_bytes=1070784512 total_nvlink_bytes=1613316576 weight_to_token_ratio=0.487155 prefetch_share_of_ingress=0.164516 dedup_ratio=0.340851 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.993919450e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085063088 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542646192 ingress_bytes=718577664 link_bytes=718577664 total_nvlink_bytes=1261223856 weight_to_token_ratio=0.162351 prefetch_share_of_ingress=0.245152 dedup_ratio=0.339600 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086496944 prefetch_bytes=0 reduce_bytes=352321536 weight_movement_bytes=352321536 egress_bytes=543363248 ingress_bytes=895455232 link_bytes=895455232 total_nvlink_bytes=1438818480 weight_to_token_ratio=0.324273 prefetch_share_of_ingress=0.000000 dedup_ratio=0.339783 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=2 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.521263277e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43168 payload_rows_remote=37766 dup_rows=22368 dup_rows_remote=19518 dedup_ratio=0.341309 router_tokens_per_expert=[85, 104, 89, 83, 85, 82, 73, 81, 66, 77, 100, 76, 87, 88, 79, 102, 78, 74, 87, 85, 85, 93, 78, 91, 83, 76, 82, 89, 99, 92, 76, 81, 96, 86, 74, 78, 93, 98, 90, 86, 89, 90, 93, 85, 71, 105, 73, 92, 82, 92, 74, 80, 85, 97, 80, 77, 80, 91, 82, 89, 88, 74, 82, 72, 83, 88, 79, 95, 97, 82, 97, 75, 88, 85, 70, 79, 82, 86, 89, 88, 96, 89, 86, 77, 96, 79, 83, 79, 104, 76, 86, 94, 89, 79, 63, 84, 85, 77, 87, 83, 90, 79, 83, 95, 84, 90, 84, 91, 83, 88, 85, 88, 92, 95, 87, 95, 78, 70, 90, 88, 82, 82, 85, 87, 92, 87, 88, 84, 96, 87, 87, 81, 89, 91, 93, 86, 82, 76, 90, 82, 74, 89, 85, 72, 88, 90, 81, 97, 77, 68, 105, 81, 91, 85, 81, 80, 86, 78, 92, 90, 71, 82, 104, 82, 69, 81, 90, 78, 90, 90, 74, 93, 79, 100, 84, 75, 86, 83, 94, 83, 104, 87, 70, 88, 74, 76, 76, 78, 97, 83, 81, 91, 82, 97, 87, 74, 74, 81, 82, 91, 72, 82, 85, 76, 95, 97, 96, 84, 92, 78, 78, 95, 86, 88, 96, 99, 106, 93, 76, 107, 70, 73, 78, 88, 91, 93, 98, 64, 90, 80, 88, 100, 82, 78, 102, 77, 87, 89, 75, 74, 87, 73, 91, 97, 79, 101, 63, 85, 73, 79, 90, 86, 90, 82, 91, 95, 87, 88, 80, 79, 86, 75, 81, 68, 84, 94, 74, 87, 81, 96, 86, 77, 95, 88, 81, 94, 99, 55, 90, 74, 80, 80, 91, 83, 95, 102, 99, 91, 82, 108, 80, 71, 77, 95, 84, 87, 92, 80, 84, 94, 99, 78, 84, 83, 77, 78, 88, 84, 95, 87, 63, 82, 94, 76, 74, 73, 74, 92, 81, 71, 83, 88, 89, 80, 78, 67, 94, 82, 94, 85, 91, 81, 100, 92, 99, 62, 82, 83, 77, 93, 81, 87, 97, 84, 102, 81, 67, 104, 102, 81, 65, 88, 93, 91, 78, 86, 81, 84, 98, 84, 78, 75, 102, 92, 95, 68, 108, 98, 102, 87, 92, 82, 100, 68, 100, 82, 93, 90, 90, 88, 91, 83, 83, 82] payload_rows_per_dst=[5384, 5400, 5372, 5360, 5402, 5390, 5340, 5520] dup_rows_per_dst=[2826, 2696, 2856, 2766, 2850, 2774, 2724, 2876] combine_rows_per_src=[5384, 5400, 5372, 5360, 5402, 5390, 5340, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541413376 dispatch_meta_bytes=229136 combine_payload_bytes=541413376 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541642512 ingress_bytes=717574144 link_bytes=717574144 token_bytes=1083055888 weight_movement_bytes=176160768 total_nvlink_bytes=1259216656 weight_to_token_ratio=0.162652 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.027173514e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083055888 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541642512 ingress_bytes=717574144 link_bytes=717574144 total_nvlink_bytes=1259216656 weight_to_token_ratio=0.162652 prefetch_share_of_ingress=0.245495 dedup_ratio=0.341309 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.027173514e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 120.2 | throughput per GPU (TFLOP/s/GPU): 294.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.132326E+01 | loss scale: 1.0 | grad norm: 25.632 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43464 payload_rows_remote=38002 dup_rows=22072 dup_rows_remote=19140 dedup_ratio=0.336792 router_tokens_per_expert=[79, 74, 89, 84, 93, 101, 95, 85, 73, 83, 86, 90, 83, 70, 73, 72, 95, 91, 70, 78, 87, 89, 88, 85, 96, 70, 66, 77, 85, 76, 79, 69, 76, 79, 79, 79, 87, 86, 100, 77, 84, 85, 100, 82, 71, 86, 86, 77, 75, 81, 69, 83, 88, 96, 85, 86, 73, 84, 77, 78, 89, 69, 90, 88, 90, 94, 85, 87, 77, 74, 82, 81, 90, 70, 97, 93, 83, 89, 83, 83, 103, 90, 87, 96, 68, 88, 76, 87, 81, 84, 87, 71, 92, 68, 95, 75, 77, 90, 112, 91, 93, 93, 76, 82, 83, 86, 84, 80, 81, 97, 78, 96, 84, 92, 91, 85, 82, 96, 79, 98, 86, 80, 79, 87, 81, 98, 71, 76, 67, 86, 84, 73, 110, 90, 77, 74, 82, 105, 79, 82, 84, 98, 84, 94, 85, 79, 68, 80, 91, 93, 91, 95, 92, 84, 84, 79, 113, 65, 82, 72, 82, 95, 92, 96, 98, 85, 82, 92, 87, 78, 87, 77, 87, 77, 91, 90, 92, 89, 77, 57, 85, 106, 73, 98, 84, 84, 85, 88, 82, 84, 93, 89, 97, 77, 87, 78, 88, 95, 85, 93, 100, 93, 78, 79, 79, 86, 88, 86, 96, 80, 117, 84, 74, 80, 89, 107, 79, 102, 80, 92, 78, 92, 79, 82, 77, 95, 91, 100, 91, 76, 78, 86, 78, 78, 91, 88, 75, 108, 90, 95, 77, 91, 86, 97, 88, 72, 92, 85, 77, 86, 86, 75, 92, 92, 74, 97, 98, 78, 102, 91, 65, 77, 92, 94, 81, 86, 93, 80, 88, 105, 84, 91, 106, 89, 96, 71, 97, 81, 76, 91, 86, 81, 87, 106, 85, 85, 106, 88, 71, 88, 101, 92, 88, 59, 94, 87, 98, 73, 102, 86, 92, 96, 87, 88, 84, 97, 88, 72, 77, 87, 80, 98, 70, 86, 102, 88, 101, 88, 77, 84, 105, 73, 79, 93, 93, 87, 82, 86, 73, 66, 101, 82, 74, 100, 76, 75, 72, 73, 80, 62, 96, 74, 87, 67, 80, 79, 75, 81, 78, 78, 62, 86, 68, 79, 87, 95, 94, 69, 85, 67, 79, 81, 80, 102, 93, 103, 74, 95, 99, 101, 73, 86, 88, 70, 90, 87, 104, 90, 85, 83, 88, 86, 88, 113] payload_rows_per_dst=[5340, 5356, 5514, 5428, 5462, 5484, 5476, 5404] dup_rows_per_dst=[2590, 2678, 2752, 2802, 2932, 2922, 2776, 2620] combine_rows_per_src=[5340, 5356, 5514, 5428, 5462, 5484, 5476, 5404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43274 payload_rows_remote=37988 dup_rows=22262 dup_rows_remote=19536 dedup_ratio=0.339691 router_tokens_per_expert=[74, 96, 86, 78, 67, 108, 74, 70, 72, 84, 72, 88, 69, 84, 90, 88, 73, 97, 86, 71, 102, 101, 81, 95, 77, 73, 84, 78, 89, 83, 96, 86, 105, 88, 76, 85, 80, 98, 87, 81, 87, 74, 94, 93, 70, 83, 101, 96, 97, 83, 86, 81, 89, 72, 100, 90, 89, 70, 76, 79, 82, 91, 83, 75, 92, 77, 74, 87, 97, 81, 89, 91, 89, 100, 96, 84, 82, 77, 72, 94, 85, 78, 85, 80, 85, 82, 82, 88, 96, 75, 91, 76, 84, 73, 94, 83, 90, 93, 108, 91, 73, 96, 73, 80, 93, 73, 86, 79, 91, 67, 94, 87, 91, 84, 93, 74, 71, 87, 82, 87, 84, 79, 97, 73, 80, 84, 64, 82, 109, 94, 100, 85, 81, 79, 73, 80, 77, 68, 86, 94, 71, 97, 82, 88, 98, 81, 76, 72, 85, 94, 82, 91, 89, 64, 76, 75, 83, 103, 85, 83, 90, 103, 78, 66, 78, 74, 74, 82, 83, 87, 76, 84, 85, 75, 90, 81, 89, 70, 75, 96, 83, 65, 90, 87, 100, 88, 94, 88, 72, 96, 86, 84, 88, 108, 79, 94, 75, 87, 95, 82, 90, 95, 99, 101, 65, 75, 83, 81, 87, 84, 85, 81, 85, 79, 91, 98, 74, 72, 77, 94, 86, 89, 95, 77, 91, 87, 83, 88, 84, 95, 84, 99, 96, 100, 77, 94, 85, 79, 93, 83, 74, 79, 84, 94, 85, 93, 90, 82, 74, 97, 90, 82, 84, 82, 85, 92, 118, 86, 88, 81, 89, 84, 77, 97, 79, 89, 92, 77, 76, 70, 99, 99, 84, 93, 95, 70, 79, 85, 79, 85, 85, 78, 74, 99, 84, 80, 88, 82, 79, 73, 116, 82, 81, 85, 84, 85, 82, 89, 84, 86, 73, 88, 90, 91, 82, 80, 87, 78, 75, 85, 90, 82, 107, 85, 85, 104, 87, 106, 79, 90, 80, 71, 89, 86, 97, 75, 83, 104, 87, 76, 75, 92, 80, 85, 99, 101, 79, 78, 89, 86, 77, 63, 76, 96, 86, 105, 95, 66, 101, 90, 97, 71, 69, 91, 92, 85, 87, 78, 82, 73, 106, 77, 88, 95, 78, 87, 86, 102, 103, 92, 79, 93, 87, 89, 78, 81, 99, 86, 94, 83, 78, 95, 92, 93] payload_rows_per_dst=[5446, 5454, 5412, 5286, 5414, 5436, 5410, 5416] dup_rows_per_dst=[2854, 2670, 2688, 2726, 2764, 2780, 2890, 2890] combine_rows_per_src=[5446, 5454, 5412, 5286, 5414, 5436, 5410, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43202 payload_rows_remote=37826 dup_rows=22334 dup_rows_remote=19468 dedup_ratio=0.340790 router_tokens_per_expert=[78, 83, 93, 80, 77, 65, 103, 85, 76, 63, 115, 100, 79, 79, 86, 95, 82, 104, 86, 91, 91, 78, 88, 85, 82, 89, 93, 76, 69, 100, 98, 91, 84, 82, 84, 87, 68, 90, 73, 74, 85, 81, 62, 79, 80, 71, 87, 85, 79, 67, 92, 92, 95, 90, 81, 91, 73, 83, 79, 85, 83, 87, 66, 103, 75, 91, 93, 82, 77, 86, 105, 87, 76, 87, 100, 84, 98, 78, 101, 85, 82, 76, 90, 78, 97, 84, 76, 85, 87, 87, 96, 93, 83, 74, 102, 80, 95, 92, 93, 98, 77, 91, 74, 75, 104, 86, 104, 94, 85, 74, 89, 99, 89, 69, 93, 73, 70, 94, 86, 82, 101, 89, 86, 80, 62, 89, 78, 65, 85, 98, 110, 78, 80, 88, 83, 74, 89, 90, 81, 83, 74, 77, 72, 75, 98, 94, 94, 86, 84, 73, 72, 92, 91, 66, 92, 76, 84, 66, 100, 93, 82, 103, 80, 91, 78, 77, 69, 96, 78, 118, 99, 79, 88, 79, 95, 74, 85, 90, 96, 75, 83, 82, 84, 81, 72, 80, 83, 78, 85, 94, 95, 90, 82, 87, 81, 78, 74, 89, 102, 83, 81, 94, 94, 105, 77, 101, 74, 74, 80, 82, 88, 94, 97, 95, 74, 84, 95, 100, 92, 86, 84, 71, 98, 90, 115, 85, 94, 86, 75, 85, 75, 90, 67, 73, 84, 101, 78, 84, 79, 96, 98, 88, 76, 87, 81, 73, 99, 85, 72, 76, 102, 84, 90, 86, 76, 72, 85, 82, 93, 78, 95, 80, 94, 89, 82, 82, 98, 70, 85, 99, 98, 75, 110, 89, 66, 70, 103, 73, 83, 85, 88, 95, 93, 87, 65, 95, 72, 97, 85, 86, 92, 83, 97, 98, 101, 88, 86, 87, 88, 84, 94, 81, 66, 95, 76, 85, 87, 84, 70, 86, 96, 86, 85, 85, 87, 85, 84, 97, 74, 83, 95, 90, 77, 97, 70, 94, 84, 98, 86, 73, 91, 82, 89, 69, 80, 81, 74, 88, 86, 75, 83, 91, 96, 89, 65, 94, 85, 86, 81, 73, 85, 96, 86, 91, 80, 80, 85, 75, 83, 92, 89, 99, 89, 75, 93, 99, 67, 69, 81, 96, 82, 87, 99, 93, 84, 68, 102, 81, 77, 74, 91, 88, 87, 82] payload_rows_per_dst=[5442, 5376, 5384, 5384, 5404, 5434, 5364, 5414] dup_rows_per_dst=[2798, 2866, 2762, 2816, 2836, 2768, 2870, 2618] combine_rows_per_src=[5442, 5376, 5384, 5384, 5404, 5434, 5364, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43198 payload_rows_remote=37786 dup_rows=22338 dup_rows_remote=19554 dedup_ratio=0.340851 router_tokens_per_expert=[93, 70, 75, 82, 86, 85, 72, 82, 79, 73, 98, 83, 89, 83, 77, 88, 99, 91, 79, 76, 89, 95, 84, 83, 94, 97, 82, 92, 79, 73, 85, 81, 94, 89, 65, 80, 86, 109, 80, 78, 88, 69, 71, 81, 96, 83, 83, 80, 73, 77, 86, 81, 98, 75, 84, 87, 82, 78, 78, 94, 91, 78, 99, 92, 78, 75, 91, 91, 83, 77, 84, 71, 87, 72, 75, 86, 77, 82, 99, 107, 109, 88, 91, 95, 89, 79, 73, 65, 77, 81, 81, 77, 95, 82, 81, 81, 91, 104, 90, 86, 97, 93, 80, 87, 75, 85, 84, 65, 100, 87, 92, 79, 105, 69, 80, 94, 96, 85, 95, 89, 82, 88, 98, 75, 76, 89, 85, 83, 74, 80, 94, 94, 82, 78, 79, 82, 85, 91, 108, 79, 105, 86, 96, 89, 95, 92, 89, 80, 96, 94, 88, 79, 95, 93, 85, 92, 74, 77, 86, 90, 76, 101, 75, 82, 84, 77, 83, 82, 80, 95, 84, 89, 94, 73, 109, 104, 90, 88, 84, 86, 76, 89, 92, 89, 76, 100, 70, 88, 78, 84, 99, 80, 91, 74, 80, 92, 67, 86, 110, 90, 91, 94, 107, 67, 70, 79, 74, 72, 101, 94, 93, 96, 83, 89, 65, 86, 73, 78, 71, 97, 77, 65, 85, 97, 102, 97, 91, 83, 76, 64, 79, 83, 69, 92, 79, 97, 88, 87, 72, 97, 85, 79, 78, 84, 83, 93, 86, 78, 79, 82, 87, 87, 64, 107, 95, 94, 93, 77, 77, 98, 68, 97, 85, 106, 98, 75, 94, 62, 89, 80, 105, 76, 77, 83, 112, 89, 101, 90, 84, 96, 89, 80, 76, 101, 86, 98, 95, 80, 80, 78, 95, 86, 94, 82, 83, 69, 78, 78, 97, 72, 101, 86, 84, 84, 82, 90, 68, 76, 74, 91, 104, 73, 108, 83, 82, 71, 87, 78, 92, 77, 95, 66, 104, 98, 80, 102, 86, 67, 84, 83, 82, 93, 84, 82, 90, 94, 82, 75, 83, 80, 106, 108, 92, 79, 67, 85, 101, 68, 79, 91, 84, 89, 96, 93, 92, 91, 63, 74, 85, 105, 73, 88, 89, 84, 79, 79, 78, 83, 87, 81, 72, 79, 96, 95, 89, 82, 90, 69, 86, 77, 94, 75, 85, 83] payload_rows_per_dst=[5394, 5394, 5508, 5380, 5352, 5432, 5326, 5412] dup_rows_per_dst=[2658, 2670, 2832, 2894, 2748, 2876, 2876, 2784] combine_rows_per_src=[5394, 5394, 5508, 5380, 5352, 5432, 5326, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43396 payload_rows_remote=37960 dup_rows=22140 dup_rows_remote=19374 dedup_ratio=0.337830 router_tokens_per_expert=[91, 98, 77, 78, 75, 83, 96, 80, 90, 87, 95, 85, 87, 73, 96, 91, 97, 79, 99, 75, 81, 96, 76, 83, 92, 81, 60, 91, 81, 85, 96, 84, 89, 98, 92, 72, 88, 88, 87, 79, 77, 76, 95, 99, 90, 75, 94, 79, 74, 83, 81, 75, 94, 96, 82, 81, 89, 82, 92, 94, 75, 69, 92, 88, 96, 78, 80, 91, 95, 93, 90, 96, 78, 85, 86, 98, 82, 70, 93, 92, 103, 83, 104, 96, 83, 66, 72, 76, 87, 102, 88, 85, 83, 96, 87, 100, 67, 71, 84, 78, 80, 85, 84, 89, 104, 93, 97, 78, 101, 82, 84, 67, 110, 80, 91, 73, 72, 85, 100, 85, 86, 104, 91, 95, 92, 93, 86, 68, 85, 93, 70, 103, 77, 78, 78, 75, 89, 94, 74, 67, 85, 102, 88, 88, 74, 98, 74, 76, 109, 70, 74, 81, 86, 92, 89, 84, 81, 86, 88, 91, 87, 93, 91, 77, 99, 74, 73, 84, 80, 84, 93, 91, 84, 82, 75, 90, 92, 80, 85, 85, 83, 80, 87, 97, 98, 84, 68, 102, 99, 74, 85, 89, 72, 76, 83, 80, 84, 87, 86, 75, 83, 71, 87, 87, 87, 83, 76, 78, 85, 79, 87, 86, 71, 77, 68, 86, 93, 78, 92, 91, 65, 86, 88, 83, 81, 82, 102, 89, 68, 69, 93, 91, 85, 67, 106, 92, 83, 93, 85, 72, 83, 77, 98, 75, 95, 85, 91, 77, 94, 80, 90, 98, 77, 92, 80, 97, 81, 76, 75, 79, 88, 104, 83, 93, 82, 78, 99, 81, 86, 101, 79, 92, 85, 84, 75, 67, 80, 96, 76, 87, 78, 96, 82, 95, 98, 80, 86, 84, 86, 69, 85, 90, 91, 70, 94, 86, 89, 79, 77, 86, 74, 73, 81, 93, 82, 81, 99, 95, 91, 71, 87, 82, 64, 87, 96, 94, 84, 89, 94, 87, 75, 81, 84, 72, 92, 82, 66, 64, 85, 83, 75, 90, 72, 72, 82, 88, 95, 88, 93, 87, 106, 65, 74, 106, 80, 91, 94, 89, 83, 79, 87, 87, 74, 89, 86, 114, 89, 70, 84, 75, 95, 90, 93, 91, 81, 97, 99, 97, 93, 88, 84, 92, 112, 111, 90, 67, 78, 86, 81, 83, 98, 67, 96, 86] payload_rows_per_dst=[5512, 5478, 5436, 5390, 5254, 5416, 5308, 5602] dup_rows_per_dst=[2894, 2844, 2766, 2806, 2508, 2814, 2630, 2878] combine_rows_per_src=[5512, 5478, 5436, 5390, 5254, 5416, 5308, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43130 payload_rows_remote=37712 dup_rows=22406 dup_rows_remote=19614 dedup_ratio=0.341888 router_tokens_per_expert=[110, 87, 97, 107, 86, 88, 81, 93, 94, 80, 86, 88, 79, 65, 87, 79, 91, 72, 81, 71, 86, 93, 81, 95, 74, 89, 83, 80, 78, 76, 74, 67, 91, 88, 86, 58, 94, 85, 88, 87, 82, 77, 83, 86, 99, 81, 98, 85, 96, 78, 90, 63, 89, 83, 89, 81, 92, 85, 88, 73, 84, 90, 107, 96, 81, 82, 75, 75, 86, 74, 86, 81, 108, 67, 89, 97, 88, 103, 90, 71, 79, 84, 89, 84, 87, 98, 95, 89, 103, 72, 84, 92, 98, 100, 91, 82, 84, 95, 77, 70, 70, 91, 91, 73, 97, 80, 89, 92, 71, 83, 78, 88, 78, 89, 72, 94, 103, 85, 83, 83, 88, 80, 93, 85, 108, 80, 75, 102, 90, 95, 97, 86, 101, 65, 80, 67, 86, 79, 82, 77, 89, 87, 103, 91, 89, 89, 70, 92, 92, 79, 69, 100, 90, 83, 88, 89, 86, 74, 85, 99, 87, 99, 70, 76, 85, 81, 83, 106, 83, 88, 88, 86, 91, 84, 74, 94, 94, 93, 81, 89, 105, 94, 72, 98, 92, 83, 78, 98, 93, 85, 83, 74, 93, 86, 92, 72, 101, 92, 95, 101, 88, 85, 87, 92, 83, 93, 81, 89, 112, 79, 95, 75, 79, 79, 96, 80, 92, 77, 77, 62, 93, 84, 95, 97, 85, 74, 110, 110, 65, 77, 99, 71, 86, 90, 79, 70, 75, 83, 95, 70, 69, 87, 87, 74, 89, 71, 87, 80, 72, 101, 99, 90, 92, 88, 74, 77, 88, 80, 87, 79, 86, 85, 86, 94, 87, 83, 78, 76, 90, 63, 87, 56, 76, 70, 83, 75, 85, 81, 85, 95, 64, 85, 76, 78, 88, 86, 80, 80, 85, 80, 107, 94, 85, 75, 77, 80, 84, 84, 103, 83, 89, 82, 65, 91, 73, 82, 101, 99, 105, 81, 82, 80, 85, 82, 78, 87, 83, 108, 77, 85, 102, 70, 90, 99, 89, 88, 70, 81, 80, 82, 75, 86, 75, 92, 78, 96, 65, 79, 77, 91, 89, 74, 89, 93, 84, 79, 93, 106, 85, 97, 91, 93, 98, 76, 88, 91, 86, 70, 80, 71, 91, 80, 95, 72, 72, 85, 85, 94, 91, 97, 73, 102, 94, 81, 91, 73, 84, 81, 82, 94, 95, 76, 79, 88] payload_rows_per_dst=[5366, 5426, 5378, 5414, 5486, 5246, 5418, 5396] dup_rows_per_dst=[2766, 2902, 2826, 2908, 2796, 2612, 2792, 2804] combine_rows_per_src=[5366, 5426, 5378, 5414, 5486, 5246, 5418, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43184 payload_rows_remote=37852 dup_rows=22352 dup_rows_remote=19724 dedup_ratio=0.341064 router_tokens_per_expert=[76, 92, 80, 90, 82, 75, 85, 83, 89, 70, 87, 104, 86, 91, 95, 77, 80, 80, 90, 85, 99, 90, 73, 83, 90, 82, 83, 99, 84, 96, 86, 95, 63, 91, 80, 90, 69, 90, 85, 94, 95, 76, 111, 87, 84, 75, 96, 88, 76, 89, 96, 99, 91, 94, 83, 75, 89, 80, 98, 87, 94, 89, 83, 100, 81, 90, 102, 86, 83, 70, 82, 81, 88, 71, 76, 103, 81, 91, 81, 106, 83, 80, 77, 99, 77, 85, 99, 77, 82, 77, 84, 94, 77, 78, 80, 78, 70, 92, 96, 87, 83, 97, 94, 83, 93, 107, 86, 81, 104, 69, 87, 82, 92, 90, 93, 83, 88, 89, 57, 68, 87, 114, 73, 68, 79, 88, 70, 68, 87, 91, 81, 84, 86, 93, 77, 60, 84, 87, 89, 88, 80, 86, 74, 86, 93, 105, 81, 77, 88, 89, 88, 91, 94, 67, 97, 102, 92, 84, 74, 76, 80, 93, 73, 76, 87, 72, 94, 90, 88, 96, 83, 87, 88, 86, 80, 87, 89, 75, 92, 79, 101, 96, 82, 95, 84, 105, 71, 70, 85, 96, 76, 87, 83, 86, 93, 68, 91, 93, 81, 98, 84, 79, 108, 88, 92, 77, 75, 79, 89, 103, 98, 99, 99, 73, 85, 84, 80, 82, 85, 85, 89, 66, 66, 80, 110, 87, 77, 103, 94, 66, 75, 87, 85, 90, 80, 86, 63, 82, 97, 100, 75, 90, 63, 84, 94, 83, 75, 66, 73, 79, 79, 86, 85, 91, 102, 83, 88, 70, 80, 77, 78, 80, 90, 77, 79, 69, 87, 84, 86, 83, 73, 71, 71, 81, 91, 88, 100, 98, 83, 82, 76, 94, 72, 90, 96, 91, 91, 96, 77, 89, 84, 89, 90, 96, 77, 79, 82, 79, 93, 104, 103, 92, 79, 94, 81, 81, 88, 109, 76, 97, 86, 75, 84, 91, 69, 89, 99, 96, 66, 82, 82, 78, 90, 81, 111, 80, 77, 82, 88, 66, 78, 102, 92, 95, 85, 79, 104, 94, 89, 77, 78, 83, 83, 80, 69, 74, 87, 95, 86, 84, 82, 93, 76, 73, 82, 99, 83, 92, 91, 73, 85, 97, 87, 85, 82, 85, 92, 69, 84, 99, 80, 91, 97, 80, 94, 77, 81, 80, 98, 94, 85, 87, 78, 67] payload_rows_per_dst=[5400, 5390, 5402, 5362, 5440, 5332, 5464, 5394] dup_rows_per_dst=[2862, 2854, 2700, 2920, 2800, 2628, 2820, 2768] combine_rows_per_src=[5400, 5390, 5402, 5362, 5440, 5332, 5464, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43338 payload_rows_remote=37918 dup_rows=22198 dup_rows_remote=19404 dedup_ratio=0.338715 router_tokens_per_expert=[99, 91, 88, 76, 94, 58, 78, 91, 86, 71, 102, 73, 76, 68, 82, 95, 91, 81, 75, 91, 75, 85, 80, 79, 93, 88, 100, 74, 93, 95, 88, 88, 79, 82, 83, 80, 92, 80, 78, 81, 87, 79, 81, 82, 75, 79, 85, 90, 63, 88, 84, 79, 90, 88, 83, 83, 88, 83, 89, 90, 95, 81, 85, 82, 87, 80, 79, 83, 85, 75, 90, 107, 82, 86, 70, 75, 88, 54, 94, 109, 76, 90, 66, 66, 78, 84, 78, 104, 99, 78, 81, 79, 92, 103, 81, 79, 73, 71, 87, 79, 92, 94, 77, 80, 81, 69, 112, 76, 105, 90, 74, 98, 92, 91, 93, 68, 98, 82, 87, 103, 84, 86, 90, 93, 81, 83, 84, 90, 100, 88, 75, 83, 98, 87, 97, 70, 85, 77, 94, 87, 86, 88, 73, 97, 93, 70, 81, 79, 93, 84, 100, 81, 79, 66, 92, 84, 76, 80, 73, 96, 92, 65, 82, 88, 83, 90, 100, 87, 90, 101, 87, 80, 75, 91, 65, 80, 78, 75, 76, 79, 76, 79, 101, 77, 92, 81, 78, 91, 79, 82, 106, 77, 99, 80, 75, 88, 92, 92, 82, 66, 94, 93, 83, 98, 84, 85, 85, 79, 82, 84, 90, 91, 79, 87, 83, 84, 105, 82, 84, 71, 96, 82, 105, 101, 89, 71, 81, 91, 80, 90, 77, 84, 89, 96, 78, 84, 94, 80, 100, 113, 86, 85, 65, 86, 81, 89, 78, 86, 74, 85, 95, 115, 89, 92, 83, 102, 89, 83, 86, 80, 94, 81, 95, 80, 85, 84, 78, 88, 99, 69, 67, 82, 94, 90, 97, 86, 76, 96, 80, 101, 101, 94, 85, 95, 90, 91, 84, 87, 91, 99, 84, 76, 88, 71, 92, 65, 71, 91, 91, 81, 93, 87, 82, 65, 84, 98, 83, 74, 101, 70, 75, 78, 103, 73, 86, 80, 84, 89, 64, 81, 99, 95, 95, 94, 80, 82, 75, 94, 89, 107, 93, 95, 87, 65, 92, 66, 90, 90, 79, 99, 89, 71, 95, 82, 73, 105, 87, 101, 87, 87, 70, 94, 56, 80, 91, 85, 90, 87, 79, 68, 81, 77, 81, 77, 85, 89, 87, 79, 94, 100, 94, 101, 82, 89, 92, 89, 97, 75, 84, 66, 98, 83, 102, 113] payload_rows_per_dst=[5420, 5472, 5410, 5330, 5474, 5472, 5416, 5344] dup_rows_per_dst=[2794, 2706, 2766, 2690, 2866, 2884, 2700, 2792] combine_rows_per_src=[5420, 5472, 5410, 5330, 5474, 5472, 5416, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544796672 dispatch_meta_bytes=228568 combine_payload_bytes=544796672 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=545025240 ingress_bytes=897118208 link_bytes=897118208 token_bytes=1089821912 weight_movement_bytes=352321536 total_nvlink_bytes=1442143448 weight_to_token_ratio=0.323284 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.535231234e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544595968 dispatch_meta_bytes=230096 combine_payload_bytes=544595968 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=544826064 ingress_bytes=896917504 link_bytes=896917504 token_bytes=1089422032 weight_movement_bytes=352321536 total_nvlink_bytes=1441743568 weight_to_token_ratio=0.323402 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.533545446e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542273536 dispatch_meta_bytes=229176 combine_payload_bytes=542273536 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542502712 ingress_bytes=718434304 link_bytes=718434304 token_bytes=1084776248 weight_movement_bytes=176160768 total_nvlink_bytes=1260937016 weight_to_token_ratio=0.162394 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.034398320e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541700096 dispatch_meta_bytes=229360 combine_payload_bytes=541700096 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=541929456 ingress_bytes=894021632 link_bytes=894021632 token_bytes=1083629552 weight_movement_bytes=352321536 total_nvlink_bytes=1435951088 weight_to_token_ratio=0.325131 gemm_rows_defined=35968 gemm_rows_charged=45056 gemm_tail_pad_rows=9088 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.201705 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.509221934e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540639232 dispatch_meta_bytes=229304 combine_payload_bytes=540639232 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=540868536 ingress_bytes=716800000 link_bytes=716800000 token_bytes=1081507768 weight_movement_bytes=176160768 total_nvlink_bytes=1257668536 weight_to_token_ratio=0.162884 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.020671189e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544194560 dispatch_meta_bytes=229336 combine_payload_bytes=544194560 prefetch_bytes=0 reduce_bytes=528482304 egress_bytes=544423896 ingress_bytes=1072676864 link_bytes=1072676864 token_bytes=1088618456 weight_movement_bytes=528482304 total_nvlink_bytes=1617100760 weight_to_token_ratio=0.485461 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.190341 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.009814022e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542646272 dispatch_meta_bytes=230304 combine_payload_bytes=542646272 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542876576 ingress_bytes=894967808 link_bytes=894967808 token_bytes=1085522848 weight_movement_bytes=352321536 total_nvlink_bytes=1437844384 weight_to_token_ratio=0.324564 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.517169220e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1089821912 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=545025240 ingress_bytes=897118208 link_bytes=897118208 total_nvlink_bytes=1442143448 weight_to_token_ratio=0.323284 prefetch_share_of_ingress=0.196363 dedup_ratio=0.336792 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.535231234e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1089422032 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=544826064 ingress_bytes=896917504 link_bytes=896917504 total_nvlink_bytes=1441743568 weight_to_token_ratio=0.323402 prefetch_share_of_ingress=0.196407 dedup_ratio=0.339691 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.533545446e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543592448 dispatch_meta_bytes=229288 combine_payload_bytes=543592448 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543821736 ingress_bytes=719753216 link_bytes=719753216 token_bytes=1087414184 weight_movement_bytes=176160768 total_nvlink_bytes=1263574952 weight_to_token_ratio=0.162000 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084776248 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542502712 ingress_bytes=718434304 link_bytes=718434304 total_nvlink_bytes=1260937016 weight_to_token_ratio=0.162394 prefetch_share_of_ingress=0.245201 dedup_ratio=0.340790 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.034398320e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1081507768 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=540868536 ingress_bytes=716800000 link_bytes=716800000 total_nvlink_bytes=1257668536 weight_to_token_ratio=0.162884 prefetch_share_of_ingress=0.245760 dedup_ratio=0.341888 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.020671189e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083629552 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=541929456 ingress_bytes=894021632 link_bytes=894021632 total_nvlink_bytes=1435951088 weight_to_token_ratio=0.325131 prefetch_share_of_ingress=0.197043 dedup_ratio=0.340851 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.201705 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.509221934e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088618456 prefetch_bytes=0 reduce_bytes=528482304 weight_movement_bytes=528482304 egress_bytes=544423896 ingress_bytes=1072676864 link_bytes=1072676864 total_nvlink_bytes=1617100760 weight_to_token_ratio=0.485461 prefetch_share_of_ingress=0.000000 dedup_ratio=0.337830 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=3 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=9.009814022e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085522848 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542876576 ingress_bytes=894967808 link_bytes=894967808 total_nvlink_bytes=1437844384 weight_to_token_ratio=0.324564 prefetch_share_of_ingress=0.196835 dedup_ratio=0.341064 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.517169220e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087414184 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543821736 ingress_bytes=719753216 link_bytes=719753216 total_nvlink_bytes=1263574952 weight_to_token_ratio=0.162000 prefetch_share_of_ingress=0.244752 dedup_ratio=0.338715 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 120.3 | throughput per GPU (TFLOP/s/GPU): 294.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.160290E+01 | loss scale: 1.0 | grad norm: 12.097 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43274 payload_rows_remote=37836 dup_rows=22262 dup_rows_remote=19408 dedup_ratio=0.339691 router_tokens_per_expert=[87, 78, 67, 84, 85, 87, 85, 75, 78, 82, 82, 84, 73, 70, 87, 81, 75, 80, 84, 94, 84, 96, 73, 80, 85, 91, 98, 70, 98, 85, 84, 98, 98, 81, 81, 78, 79, 97, 95, 77, 86, 80, 95, 76, 85, 79, 82, 105, 79, 70, 87, 94, 74, 82, 73, 90, 84, 88, 87, 80, 94, 75, 75, 88, 86, 96, 63, 73, 88, 98, 72, 75, 67, 89, 68, 88, 104, 67, 74, 77, 89, 71, 81, 67, 90, 83, 71, 78, 106, 69, 75, 76, 88, 82, 96, 97, 104, 101, 84, 93, 101, 99, 69, 94, 78, 106, 77, 77, 89, 75, 75, 83, 102, 91, 91, 97, 80, 83, 72, 81, 95, 88, 87, 87, 95, 71, 86, 78, 88, 79, 106, 83, 73, 93, 100, 83, 90, 100, 83, 91, 79, 80, 88, 75, 85, 73, 83, 83, 84, 82, 81, 61, 72, 81, 85, 80, 109, 82, 75, 105, 93, 90, 98, 81, 87, 94, 85, 89, 72, 93, 92, 83, 74, 95, 90, 80, 85, 80, 71, 96, 103, 73, 90, 93, 97, 68, 90, 87, 74, 73, 95, 81, 85, 86, 95, 105, 92, 87, 77, 102, 93, 102, 99, 75, 73, 92, 76, 91, 84, 82, 78, 79, 80, 82, 100, 100, 99, 110, 90, 96, 76, 84, 80, 88, 82, 87, 88, 105, 98, 81, 89, 72, 86, 81, 80, 83, 81, 94, 91, 98, 75, 81, 85, 93, 82, 79, 76, 71, 94, 86, 92, 75, 84, 85, 105, 76, 82, 89, 87, 84, 88, 75, 94, 94, 94, 93, 81, 69, 99, 97, 92, 80, 86, 75, 70, 70, 64, 116, 93, 77, 96, 89, 89, 76, 87, 99, 84, 100, 99, 87, 83, 83, 86, 75, 79, 84, 72, 92, 81, 86, 70, 83, 95, 85, 75, 89, 66, 99, 89, 86, 94, 79, 96, 84, 99, 62, 77, 80, 98, 105, 94, 93, 88, 71, 85, 86, 81, 84, 105, 84, 103, 76, 92, 84, 86, 81, 88, 100, 94, 96, 89, 79, 83, 81, 75, 82, 83, 84, 98, 117, 76, 86, 76, 93, 85, 86, 98, 79, 67, 75, 76, 88, 90, 78, 85, 83, 81, 67, 86, 95, 72, 83, 89, 85, 73, 85, 82, 92, 79, 85, 104, 102, 95, 79] payload_rows_per_dst=[5338, 5284, 5484, 5370, 5438, 5484, 5454, 5422] dup_rows_per_dst=[2730, 2564, 2876, 2776, 2854, 2732, 2768, 2962] combine_rows_per_src=[5338, 5284, 5484, 5370, 5438, 5484, 5454, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43280 payload_rows_remote=37830 dup_rows=22256 dup_rows_remote=19392 dedup_ratio=0.339600 router_tokens_per_expert=[93, 87, 83, 78, 84, 116, 83, 98, 75, 91, 76, 85, 69, 65, 91, 91, 87, 104, 78, 101, 88, 78, 86, 74, 97, 79, 71, 89, 89, 101, 71, 101, 87, 80, 82, 87, 84, 83, 91, 83, 74, 98, 82, 80, 79, 80, 77, 80, 85, 83, 76, 85, 96, 73, 81, 76, 61, 81, 79, 87, 77, 78, 86, 74, 109, 93, 91, 84, 104, 82, 83, 86, 60, 93, 92, 75, 97, 85, 102, 85, 89, 66, 95, 65, 79, 84, 90, 74, 104, 87, 84, 84, 84, 99, 88, 67, 106, 71, 85, 76, 85, 88, 76, 92, 90, 89, 82, 82, 69, 82, 92, 86, 69, 102, 92, 77, 79, 79, 82, 82, 69, 81, 69, 89, 96, 82, 90, 81, 95, 69, 66, 97, 82, 94, 99, 78, 100, 80, 88, 78, 76, 81, 85, 92, 85, 104, 95, 86, 76, 83, 89, 82, 102, 54, 75, 90, 87, 64, 85, 90, 81, 87, 69, 90, 93, 99, 77, 92, 98, 78, 92, 81, 88, 93, 88, 101, 86, 97, 77, 64, 95, 98, 88, 87, 90, 95, 102, 95, 81, 95, 94, 97, 93, 96, 85, 72, 76, 107, 68, 78, 88, 85, 81, 79, 105, 95, 85, 91, 87, 88, 101, 94, 90, 70, 82, 88, 80, 88, 94, 78, 108, 82, 90, 96, 69, 82, 74, 71, 82, 82, 80, 97, 74, 91, 96, 81, 92, 89, 91, 96, 85, 105, 66, 83, 84, 91, 89, 74, 83, 89, 85, 92, 91, 91, 78, 82, 80, 103, 97, 70, 98, 99, 78, 106, 89, 87, 75, 81, 85, 96, 72, 92, 90, 94, 83, 78, 70, 87, 84, 83, 83, 98, 81, 70, 85, 92, 77, 90, 93, 90, 86, 92, 83, 83, 76, 90, 79, 111, 77, 78, 87, 89, 81, 87, 77, 67, 72, 84, 82, 91, 85, 100, 87, 82, 66, 86, 92, 75, 90, 94, 86, 81, 102, 89, 88, 80, 80, 97, 84, 87, 79, 92, 78, 87, 85, 97, 82, 102, 79, 94, 92, 85, 69, 69, 94, 68, 71, 73, 98, 78, 100, 98, 82, 72, 93, 87, 75, 79, 88, 75, 84, 71, 91, 76, 100, 96, 77, 100, 88, 88, 78, 71, 80, 76, 85, 103, 87, 87, 81, 91, 93, 85, 79, 77] payload_rows_per_dst=[5364, 5426, 5356, 5450, 5448, 5506, 5412, 5318] dup_rows_per_dst=[2808, 2726, 2704, 2864, 2846, 2736, 2796, 2776] combine_rows_per_src=[5364, 5426, 5356, 5450, 5448, 5506, 5412, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43082 payload_rows_remote=37542 dup_rows=22454 dup_rows_remote=19432 dedup_ratio=0.342621 router_tokens_per_expert=[78, 93, 73, 83, 77, 73, 81, 79, 90, 72, 85, 86, 79, 69, 90, 95, 84, 83, 102, 92, 84, 74, 86, 81, 75, 92, 86, 96, 88, 84, 90, 87, 69, 90, 88, 78, 83, 97, 85, 98, 79, 87, 88, 84, 96, 72, 79, 95, 80, 71, 90, 70, 82, 85, 85, 87, 95, 92, 88, 88, 74, 81, 84, 75, 72, 78, 93, 88, 72, 73, 76, 105, 88, 87, 83, 79, 91, 70, 93, 89, 92, 71, 81, 79, 78, 91, 98, 79, 92, 92, 74, 93, 94, 92, 79, 84, 83, 77, 78, 92, 83, 74, 84, 99, 83, 78, 92, 96, 57, 88, 70, 86, 65, 91, 89, 91, 95, 79, 77, 79, 78, 85, 107, 90, 97, 89, 95, 88, 102, 83, 85, 83, 77, 94, 85, 67, 91, 87, 105, 90, 89, 86, 69, 87, 91, 79, 93, 69, 81, 84, 103, 79, 90, 82, 81, 91, 85, 86, 95, 90, 86, 95, 89, 83, 84, 74, 77, 96, 88, 75, 87, 82, 91, 89, 90, 87, 68, 90, 69, 82, 71, 84, 88, 91, 85, 78, 99, 97, 86, 83, 71, 73, 94, 94, 98, 97, 78, 100, 84, 93, 75, 78, 91, 85, 93, 80, 98, 84, 74, 89, 81, 85, 94, 72, 98, 85, 83, 95, 103, 87, 82, 93, 99, 78, 87, 96, 97, 72, 91, 82, 99, 85, 73, 107, 87, 85, 87, 79, 82, 77, 87, 80, 71, 71, 86, 88, 91, 97, 75, 88, 85, 97, 80, 87, 76, 83, 83, 63, 79, 94, 81, 80, 91, 86, 79, 87, 88, 90, 96, 99, 87, 78, 85, 70, 101, 90, 101, 68, 108, 73, 84, 88, 92, 78, 103, 86, 82, 94, 103, 80, 79, 84, 71, 86, 71, 89, 80, 85, 81, 72, 75, 84, 86, 94, 89, 83, 94, 81, 85, 80, 88, 99, 94, 72, 91, 68, 94, 73, 80, 94, 72, 88, 90, 71, 85, 83, 73, 95, 92, 83, 93, 75, 98, 59, 89, 97, 78, 102, 90, 89, 98, 87, 84, 91, 92, 86, 87, 84, 87, 106, 88, 85, 78, 80, 82, 88, 87, 90, 88, 92, 99, 86, 88, 91, 104, 94, 108, 92, 84, 80, 58, 84, 87, 67, 81, 67, 78, 82, 85, 92, 84, 96, 84, 88] payload_rows_per_dst=[5380, 5336, 5400, 5418, 5308, 5406, 5294, 5540] dup_rows_per_dst=[2730, 2730, 2790, 2716, 2646, 2992, 2828, 3022] combine_rows_per_src=[5380, 5336, 5400, 5418, 5308, 5406, 5294, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43142 payload_rows_remote=37740 dup_rows=22394 dup_rows_remote=19484 dedup_ratio=0.341705 router_tokens_per_expert=[79, 81, 84, 90, 89, 82, 86, 86, 88, 77, 86, 87, 96, 85, 88, 82, 89, 74, 102, 82, 84, 94, 77, 89, 94, 84, 78, 86, 73, 95, 90, 79, 82, 89, 74, 88, 88, 105, 81, 90, 97, 67, 73, 88, 71, 80, 84, 91, 82, 90, 89, 95, 74, 97, 77, 86, 85, 69, 86, 93, 80, 98, 92, 74, 86, 84, 96, 92, 92, 68, 82, 87, 87, 84, 77, 84, 79, 77, 76, 90, 90, 89, 77, 82, 83, 69, 92, 78, 107, 74, 97, 91, 89, 79, 85, 79, 90, 95, 92, 78, 85, 86, 95, 88, 87, 85, 77, 111, 92, 80, 85, 89, 72, 92, 88, 92, 86, 84, 98, 88, 84, 78, 91, 89, 83, 80, 102, 104, 76, 88, 88, 93, 91, 81, 90, 94, 76, 83, 79, 94, 85, 77, 77, 70, 88, 81, 93, 64, 87, 94, 79, 67, 98, 79, 77, 79, 75, 94, 89, 91, 79, 80, 98, 75, 98, 84, 77, 84, 83, 109, 85, 69, 78, 92, 75, 70, 87, 89, 76, 86, 83, 82, 98, 99, 94, 80, 94, 76, 93, 95, 92, 81, 83, 81, 86, 94, 85, 77, 84, 84, 94, 95, 98, 83, 94, 78, 89, 78, 97, 96, 73, 93, 87, 86, 81, 77, 96, 81, 78, 87, 79, 81, 82, 98, 78, 81, 87, 87, 98, 81, 65, 88, 91, 80, 81, 69, 78, 84, 98, 84, 89, 87, 92, 101, 84, 97, 94, 69, 84, 79, 96, 83, 94, 67, 82, 71, 95, 83, 76, 103, 86, 98, 95, 67, 89, 93, 74, 90, 88, 95, 79, 83, 81, 63, 63, 69, 78, 86, 68, 83, 80, 99, 87, 81, 89, 90, 101, 80, 98, 91, 66, 98, 90, 93, 101, 94, 82, 84, 98, 80, 88, 86, 76, 87, 98, 84, 91, 85, 80, 106, 86, 79, 84, 86, 99, 75, 88, 99, 88, 91, 71, 94, 72, 97, 71, 84, 87, 91, 104, 84, 84, 66, 83, 78, 76, 83, 80, 71, 96, 92, 75, 73, 68, 68, 114, 67, 88, 95, 93, 98, 82, 88, 86, 90, 95, 79, 95, 84, 79, 80, 87, 88, 82, 90, 78, 64, 70, 78, 84, 108, 86, 88, 88, 79, 85, 63, 75, 103, 82, 103, 76, 110, 82, 84] payload_rows_per_dst=[5434, 5332, 5400, 5362, 5268, 5484, 5402, 5460] dup_rows_per_dst=[2734, 2806, 2936, 2790, 2574, 2810, 2910, 2834] combine_rows_per_src=[5434, 5332, 5400, 5362, 5268, 5484, 5402, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43182 payload_rows_remote=37836 dup_rows=22354 dup_rows_remote=19622 dedup_ratio=0.341095 router_tokens_per_expert=[92, 74, 74, 77, 87, 98, 85, 80, 90, 81, 94, 75, 103, 82, 99, 71, 81, 108, 84, 92, 93, 71, 80, 76, 83, 87, 82, 78, 90, 78, 73, 85, 87, 89, 84, 97, 86, 84, 92, 83, 84, 79, 88, 90, 86, 81, 78, 87, 83, 72, 84, 94, 81, 87, 86, 91, 66, 85, 89, 82, 93, 75, 95, 67, 82, 89, 103, 82, 100, 78, 70, 77, 66, 77, 85, 80, 82, 91, 95, 88, 96, 104, 72, 103, 97, 92, 96, 73, 89, 107, 72, 88, 88, 74, 75, 91, 76, 86, 92, 79, 75, 100, 90, 107, 80, 85, 73, 80, 69, 106, 73, 85, 82, 102, 90, 76, 100, 60, 94, 92, 91, 76, 85, 84, 80, 77, 77, 92, 99, 75, 68, 92, 95, 84, 73, 83, 89, 82, 85, 72, 87, 75, 91, 75, 89, 93, 94, 80, 107, 84, 97, 75, 82, 77, 89, 90, 77, 93, 80, 83, 102, 97, 101, 90, 89, 90, 94, 84, 80, 90, 85, 85, 80, 82, 88, 81, 80, 94, 86, 83, 86, 88, 79, 97, 88, 73, 75, 87, 71, 86, 76, 76, 76, 86, 84, 92, 93, 81, 88, 77, 88, 84, 89, 84, 108, 84, 74, 82, 87, 86, 95, 71, 106, 88, 80, 88, 91, 83, 92, 97, 117, 86, 87, 82, 81, 83, 85, 99, 82, 95, 90, 93, 101, 90, 97, 78, 88, 88, 90, 92, 85, 82, 85, 81, 98, 82, 86, 96, 81, 94, 80, 86, 96, 78, 78, 96, 76, 76, 64, 80, 110, 97, 72, 82, 104, 86, 72, 85, 86, 101, 91, 91, 84, 94, 100, 93, 86, 90, 101, 91, 72, 103, 86, 73, 70, 88, 73, 86, 93, 94, 79, 90, 76, 86, 79, 77, 64, 106, 102, 82, 70, 82, 85, 88, 85, 87, 80, 78, 52, 90, 85, 79, 81, 78, 79, 78, 93, 85, 82, 90, 94, 82, 78, 97, 87, 90, 99, 89, 87, 91, 82, 80, 84, 84, 82, 101, 76, 81, 90, 65, 83, 79, 92, 83, 89, 68, 102, 80, 88, 114, 86, 85, 83, 70, 76, 88, 94, 78, 85, 98, 72, 76, 76, 106, 74, 79, 72, 87, 79, 85, 93, 73, 78, 82, 74, 89, 88, 78, 88, 95, 80, 78, 73, 71] payload_rows_per_dst=[5458, 5454, 5346, 5404, 5410, 5416, 5384, 5310] dup_rows_per_dst=[2698, 2880, 2732, 2712, 3066, 2878, 2740, 2648] combine_rows_per_src=[5458, 5454, 5346, 5404, 5410, 5416, 5384, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43236 payload_rows_remote=37822 dup_rows=22300 dup_rows_remote=19392 dedup_ratio=0.340271 router_tokens_per_expert=[67, 90, 94, 102, 100, 107, 95, 89, 81, 86, 82, 101, 81, 82, 91, 95, 95, 86, 74, 86, 73, 94, 81, 72, 92, 87, 82, 76, 115, 75, 78, 75, 111, 77, 83, 82, 75, 90, 81, 88, 75, 88, 64, 105, 94, 82, 71, 89, 84, 71, 83, 82, 85, 86, 92, 99, 82, 70, 67, 87, 92, 79, 87, 99, 72, 72, 90, 77, 90, 83, 89, 90, 83, 86, 107, 76, 105, 81, 80, 83, 75, 99, 70, 77, 90, 94, 84, 85, 80, 94, 75, 95, 70, 88, 80, 106, 78, 83, 81, 109, 101, 83, 76, 97, 78, 76, 71, 90, 67, 65, 83, 83, 87, 101, 78, 92, 92, 77, 97, 106, 68, 71, 84, 75, 87, 73, 93, 86, 86, 90, 67, 68, 92, 67, 65, 71, 83, 66, 91, 93, 83, 98, 82, 71, 95, 82, 92, 96, 92, 86, 91, 85, 94, 86, 102, 93, 78, 62, 92, 98, 83, 75, 80, 90, 82, 78, 95, 88, 96, 100, 89, 81, 82, 95, 104, 82, 79, 76, 90, 88, 93, 69, 84, 77, 80, 81, 90, 86, 75, 101, 82, 91, 85, 91, 58, 75, 94, 75, 90, 96, 79, 92, 86, 81, 86, 97, 87, 80, 81, 94, 88, 89, 79, 97, 92, 68, 111, 89, 105, 89, 83, 91, 92, 81, 80, 89, 107, 91, 84, 94, 88, 84, 92, 72, 77, 78, 82, 107, 85, 73, 76, 85, 91, 97, 76, 88, 103, 70, 81, 78, 71, 67, 101, 92, 84, 77, 85, 77, 69, 86, 88, 85, 91, 67, 75, 89, 84, 79, 87, 81, 74, 87, 83, 85, 90, 97, 76, 85, 92, 77, 87, 75, 85, 86, 83, 97, 71, 99, 88, 100, 100, 83, 100, 100, 92, 95, 87, 93, 91, 91, 90, 77, 92, 93, 83, 100, 79, 87, 85, 79, 92, 84, 77, 85, 109, 93, 88, 95, 89, 82, 94, 86, 87, 94, 74, 83, 78, 89, 81, 71, 99, 87, 98, 74, 94, 100, 90, 83, 82, 75, 85, 72, 82, 87, 81, 88, 82, 83, 87, 79, 83, 93, 87, 69, 80, 66, 91, 79, 79, 78, 72, 69, 83, 89, 83, 95, 95, 102, 91, 84, 75, 80, 81, 82, 89, 74, 94, 100, 66, 88, 77, 82, 80, 98] payload_rows_per_dst=[5412, 5414, 5362, 5440, 5438, 5344, 5508, 5318] dup_rows_per_dst=[2910, 2908, 2638, 2712, 2890, 2674, 2906, 2662] combine_rows_per_src=[5412, 5414, 5362, 5440, 5438, 5344, 5508, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43322 payload_rows_remote=37902 dup_rows=22214 dup_rows_remote=19458 dedup_ratio=0.338959 router_tokens_per_expert=[61, 78, 88, 86, 70, 86, 68, 86, 93, 93, 91, 91, 91, 99, 69, 66, 69, 87, 78, 69, 85, 92, 98, 78, 77, 94, 95, 89, 95, 92, 95, 87, 88, 72, 88, 101, 83, 71, 87, 91, 100, 75, 99, 89, 83, 80, 81, 76, 89, 89, 77, 77, 83, 74, 83, 77, 84, 87, 93, 80, 89, 88, 67, 86, 84, 77, 94, 80, 83, 71, 100, 94, 93, 86, 77, 84, 89, 66, 78, 100, 91, 93, 77, 77, 100, 73, 97, 89, 67, 74, 84, 90, 92, 109, 79, 83, 92, 78, 84, 74, 84, 80, 80, 83, 92, 87, 80, 72, 95, 85, 83, 112, 100, 81, 94, 77, 86, 79, 82, 112, 78, 99, 65, 89, 85, 93, 90, 78, 78, 79, 78, 102, 78, 86, 113, 95, 88, 92, 77, 96, 76, 84, 110, 73, 90, 107, 73, 87, 95, 86, 91, 67, 78, 83, 105, 83, 87, 97, 114, 90, 91, 79, 72, 82, 88, 77, 78, 81, 86, 82, 86, 78, 99, 84, 86, 75, 87, 85, 73, 91, 72, 79, 90, 108, 64, 107, 98, 89, 99, 82, 69, 68, 95, 78, 67, 94, 100, 70, 80, 72, 80, 88, 89, 64, 88, 68, 85, 94, 102, 85, 88, 74, 88, 80, 77, 80, 79, 86, 89, 89, 72, 81, 76, 103, 96, 100, 92, 100, 82, 78, 99, 74, 92, 78, 86, 74, 82, 85, 83, 92, 75, 87, 85, 72, 80, 76, 93, 93, 66, 83, 105, 79, 79, 85, 94, 95, 90, 78, 77, 79, 72, 81, 98, 80, 101, 93, 85, 74, 100, 98, 65, 86, 76, 87, 76, 85, 82, 92, 95, 99, 76, 105, 92, 79, 83, 79, 95, 83, 90, 106, 106, 79, 73, 83, 77, 77, 85, 96, 83, 89, 83, 104, 90, 81, 95, 81, 80, 103, 94, 100, 94, 78, 65, 79, 76, 87, 98, 78, 82, 94, 90, 93, 70, 93, 76, 94, 93, 90, 95, 89, 80, 80, 94, 73, 85, 90, 101, 111, 92, 83, 74, 81, 89, 67, 113, 77, 83, 92, 82, 107, 78, 89, 90, 86, 88, 78, 84, 78, 79, 83, 94, 85, 66, 68, 77, 93, 75, 105, 100, 73, 71, 84, 83, 76, 75, 85, 77, 95, 81, 91, 83, 74, 88, 85] payload_rows_per_dst=[5410, 5398, 5434, 5372, 5314, 5420, 5456, 5518] dup_rows_per_dst=[2710, 2710, 2874, 2864, 2616, 2756, 2886, 2798] combine_rows_per_src=[5410, 5398, 5434, 5372, 5314, 5420, 5456, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43278 payload_rows_remote=37802 dup_rows=22258 dup_rows_remote=19314 dedup_ratio=0.339630 router_tokens_per_expert=[75, 97, 80, 84, 74, 99, 104, 84, 95, 65, 96, 75, 76, 91, 84, 76, 99, 93, 69, 86, 67, 76, 75, 77, 84, 88, 92, 91, 83, 102, 66, 85, 94, 86, 98, 95, 93, 95, 88, 98, 88, 86, 78, 82, 89, 81, 76, 91, 97, 71, 96, 90, 93, 86, 73, 70, 87, 74, 82, 82, 98, 70, 77, 91, 96, 74, 104, 92, 103, 82, 88, 81, 87, 74, 84, 74, 99, 86, 87, 81, 88, 96, 88, 79, 89, 102, 92, 96, 91, 98, 94, 93, 96, 78, 89, 92, 93, 80, 75, 93, 83, 88, 88, 85, 95, 105, 76, 75, 77, 74, 77, 86, 76, 99, 90, 93, 106, 88, 74, 83, 77, 82, 91, 75, 94, 79, 82, 74, 68, 87, 84, 73, 96, 90, 81, 65, 88, 73, 82, 82, 93, 68, 102, 72, 95, 83, 75, 86, 70, 100, 104, 82, 94, 94, 86, 114, 68, 83, 108, 78, 84, 85, 89, 87, 91, 82, 95, 80, 93, 81, 80, 98, 106, 83, 87, 68, 81, 102, 93, 72, 92, 82, 88, 88, 87, 90, 94, 88, 92, 102, 88, 76, 88, 102, 101, 95, 95, 87, 105, 108, 83, 89, 89, 86, 85, 96, 71, 94, 71, 85, 107, 87, 94, 80, 79, 84, 81, 89, 93, 94, 77, 88, 85, 79, 71, 91, 88, 100, 92, 82, 76, 85, 86, 95, 99, 87, 79, 75, 92, 78, 97, 83, 72, 92, 79, 75, 81, 72, 85, 66, 81, 89, 75, 73, 83, 73, 102, 68, 66, 71, 88, 94, 81, 77, 81, 91, 75, 82, 106, 95, 79, 88, 79, 84, 59, 84, 95, 87, 77, 79, 83, 77, 87, 86, 88, 91, 79, 94, 85, 91, 79, 83, 82, 78, 107, 86, 83, 80, 79, 80, 83, 68, 82, 88, 85, 88, 78, 93, 69, 80, 75, 78, 72, 78, 71, 86, 82, 83, 95, 104, 72, 84, 92, 90, 91, 82, 103, 95, 101, 93, 104, 102, 76, 76, 80, 93, 87, 102, 86, 89, 82, 86, 79, 87, 89, 89, 80, 79, 99, 89, 79, 78, 72, 89, 84, 81, 87, 67, 87, 82, 90, 77, 86, 78, 93, 63, 78, 74, 77, 73, 88, 62, 87, 98, 96, 76, 80, 93, 78, 93, 69, 72, 93, 71] payload_rows_per_dst=[5476, 5566, 5432, 5476, 5484, 5254, 5260, 5330] dup_rows_per_dst=[2944, 3002, 2772, 2810, 2936, 2644, 2532, 2618] combine_rows_per_src=[5476, 5566, 5432, 5476, 5484, 5254, 5260, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542416896 dispatch_meta_bytes=228976 combine_payload_bytes=542416896 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=542645872 ingress_bytes=1247059968 link_bytes=1247059968 token_bytes=1085062768 weight_movement_bytes=704643072 total_nvlink_bytes=1789705840 weight_to_token_ratio=0.649403 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.196023 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047452291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542330880 dispatch_meta_bytes=228888 combine_payload_bytes=542330880 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542559768 ingress_bytes=894652416 link_bytes=894652416 token_bytes=1084890648 weight_movement_bytes=352321536 total_nvlink_bytes=1437212184 weight_to_token_ratio=0.324753 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514520125e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=538202112 dispatch_meta_bytes=227896 combine_payload_bytes=538202112 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=538430008 ingress_bytes=714362880 link_bytes=714362880 token_bytes=1076632120 weight_movement_bytes=176160768 total_nvlink_bytes=1252792888 weight_to_token_ratio=0.163622 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=896 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.000200907e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541040640 dispatch_meta_bytes=228896 combine_payload_bytes=541040640 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=541269536 ingress_bytes=1069522944 link_bytes=1069522944 token_bytes=1082310176 weight_movement_bytes=528482304 total_nvlink_bytes=1610792480 weight_to_token_ratio=0.488291 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.983323068e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542416896 dispatch_meta_bytes=229832 combine_payload_bytes=542416896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542646728 ingress_bytes=718577664 link_bytes=718577664 token_bytes=1085063624 weight_movement_bytes=176160768 total_nvlink_bytes=1261224392 weight_to_token_ratio=0.162351 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085062768 prefetch_bytes=0 reduce_bytes=704643072 weight_movement_bytes=704643072 egress_bytes=542645872 ingress_bytes=1247059968 link_bytes=1247059968 total_nvlink_bytes=1789705840 weight_to_token_ratio=0.649403 prefetch_share_of_ingress=0.000000 dedup_ratio=0.339691 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=4 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.047452291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542216192 dispatch_meta_bytes=228856 combine_payload_bytes=542216192 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542445048 ingress_bytes=718376960 link_bytes=718376960 token_bytes=1084661240 weight_movement_bytes=176160768 total_nvlink_bytes=1260822008 weight_to_token_ratio=0.162411 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.033916666e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084890648 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542559768 ingress_bytes=894652416 link_bytes=894652416 total_nvlink_bytes=1437212184 weight_to_token_ratio=0.324753 prefetch_share_of_ingress=0.196904 dedup_ratio=0.339600 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514520125e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543363072 dispatch_meta_bytes=229440 combine_payload_bytes=543363072 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543592512 ingress_bytes=719523840 link_bytes=719523840 token_bytes=1086955584 weight_movement_bytes=176160768 total_nvlink_bytes=1263116352 weight_to_token_ratio=0.162068 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.043549740e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541929472 dispatch_meta_bytes=228464 combine_payload_bytes=541929472 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542157936 ingress_bytes=718090240 link_bytes=718090240 token_bytes=1084087408 weight_movement_bytes=176160768 total_nvlink_bytes=1260248176 weight_to_token_ratio=0.162497 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031508398e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1076632120 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=538430008 ingress_bytes=714362880 link_bytes=714362880 total_nvlink_bytes=1252792888 weight_to_token_ratio=0.163622 prefetch_share_of_ingress=0.246598 dedup_ratio=0.342621 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.000200907e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1082310176 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=541269536 ingress_bytes=1069522944 link_bytes=1069522944 total_nvlink_bytes=1610792480 weight_to_token_ratio=0.488291 prefetch_share_of_ingress=0.164710 dedup_ratio=0.341705 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.983323068e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085063624 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542646728 ingress_bytes=718577664 link_bytes=718577664 total_nvlink_bytes=1261224392 weight_to_token_ratio=0.162351 prefetch_share_of_ingress=0.245152 dedup_ratio=0.341095 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084661240 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542445048 ingress_bytes=718376960 link_bytes=718376960 total_nvlink_bytes=1260822008 weight_to_token_ratio=0.162411 prefetch_share_of_ingress=0.245221 dedup_ratio=0.340271 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.033916666e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086955584 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543592512 ingress_bytes=719523840 link_bytes=719523840 total_nvlink_bytes=1263116352 weight_to_token_ratio=0.162068 prefetch_share_of_ingress=0.244830 dedup_ratio=0.338959 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.043549740e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084087408 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542157936 ingress_bytes=718090240 link_bytes=718090240 total_nvlink_bytes=1260248176 weight_to_token_ratio=0.162497 prefetch_share_of_ingress=0.245318 dedup_ratio=0.339630 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.031508398e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:55] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 120.5 | throughput per GPU (TFLOP/s/GPU): 293.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.109047E+01 | loss scale: 1.0 | grad norm: 10.073 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43114 payload_rows_remote=37826 dup_rows=22422 dup_rows_remote=19732 dedup_ratio=0.342133 router_tokens_per_expert=[75, 92, 93, 87, 106, 73, 75, 85, 78, 89, 64, 87, 84, 85, 78, 88, 79, 104, 83, 82, 86, 82, 86, 88, 90, 79, 82, 91, 82, 85, 93, 65, 98, 78, 108, 85, 83, 92, 85, 82, 76, 85, 83, 78, 83, 85, 81, 88, 100, 60, 87, 79, 104, 82, 96, 94, 101, 77, 81, 92, 74, 93, 96, 115, 85, 74, 90, 88, 95, 83, 88, 69, 87, 90, 81, 73, 96, 78, 107, 84, 76, 87, 84, 81, 89, 88, 78, 91, 81, 89, 85, 80, 92, 84, 92, 96, 89, 71, 117, 84, 80, 77, 96, 97, 93, 99, 68, 87, 80, 85, 71, 76, 80, 76, 83, 107, 86, 93, 83, 85, 94, 91, 100, 82, 69, 83, 73, 108, 92, 99, 87, 67, 75, 81, 65, 86, 89, 90, 88, 86, 80, 77, 100, 70, 94, 74, 79, 84, 86, 80, 95, 82, 92, 70, 108, 67, 88, 88, 68, 87, 73, 86, 88, 87, 87, 85, 97, 102, 92, 82, 83, 86, 69, 79, 70, 82, 83, 89, 94, 79, 81, 102, 92, 80, 79, 90, 76, 78, 97, 78, 69, 80, 89, 93, 80, 90, 78, 98, 92, 95, 87, 77, 86, 87, 98, 85, 93, 80, 72, 78, 100, 79, 93, 94, 83, 85, 79, 71, 106, 81, 80, 78, 88, 92, 86, 76, 91, 90, 82, 92, 90, 71, 76, 70, 65, 91, 81, 91, 78, 77, 95, 80, 90, 78, 86, 84, 66, 88, 76, 85, 72, 83, 91, 96, 95, 80, 94, 72, 91, 75, 97, 72, 87, 69, 94, 80, 80, 76, 100, 85, 96, 78, 68, 80, 71, 74, 92, 57, 89, 77, 96, 84, 82, 88, 82, 82, 99, 77, 84, 98, 82, 73, 73, 96, 94, 72, 86, 88, 86, 87, 90, 101, 83, 92, 92, 86, 106, 82, 67, 84, 83, 91, 95, 100, 84, 74, 82, 83, 80, 98, 85, 82, 88, 79, 94, 102, 80, 84, 97, 95, 97, 75, 80, 78, 95, 92, 70, 96, 75, 87, 86, 90, 100, 107, 92, 92, 90, 87, 83, 98, 68, 68, 86, 82, 96, 77, 87, 81, 89, 74, 95, 74, 81, 87, 91, 97, 93, 71, 83, 85, 102, 98, 100, 86, 88, 88, 87, 77, 95, 83, 81, 84, 77, 96] payload_rows_per_dst=[5414, 5424, 5410, 5372, 5404, 5288, 5362, 5440] dup_rows_per_dst=[2718, 2920, 2780, 2706, 2744, 2690, 2988, 2876] combine_rows_per_src=[5414, 5424, 5410, 5372, 5404, 5288, 5362, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43180 payload_rows_remote=37836 dup_rows=22356 dup_rows_remote=19588 dedup_ratio=0.341125 router_tokens_per_expert=[77, 88, 80, 74, 80, 73, 82, 89, 91, 80, 89, 91, 108, 75, 81, 77, 71, 81, 69, 84, 79, 87, 103, 93, 90, 85, 76, 88, 101, 78, 72, 92, 104, 85, 71, 98, 90, 99, 91, 75, 94, 92, 81, 96, 87, 83, 96, 103, 106, 77, 88, 82, 94, 83, 89, 88, 81, 78, 91, 92, 78, 88, 88, 93, 86, 77, 83, 86, 95, 89, 74, 78, 90, 85, 65, 80, 82, 79, 87, 73, 84, 88, 81, 81, 96, 79, 83, 90, 89, 83, 70, 74, 72, 82, 72, 71, 98, 89, 84, 80, 100, 74, 87, 85, 94, 100, 83, 90, 95, 73, 98, 77, 98, 95, 95, 94, 73, 101, 74, 82, 86, 86, 95, 78, 89, 78, 73, 96, 78, 70, 84, 77, 80, 92, 71, 86, 87, 85, 81, 70, 89, 78, 96, 85, 68, 95, 66, 101, 77, 89, 95, 88, 77, 89, 76, 92, 80, 80, 98, 70, 103, 85, 83, 83, 85, 86, 93, 86, 84, 92, 83, 85, 71, 72, 87, 74, 86, 75, 92, 72, 89, 81, 90, 85, 90, 99, 75, 91, 75, 94, 89, 73, 81, 86, 66, 102, 92, 91, 74, 89, 83, 96, 74, 82, 108, 91, 93, 74, 93, 96, 70, 79, 88, 77, 65, 64, 99, 100, 70, 94, 74, 83, 70, 103, 103, 85, 76, 96, 86, 95, 77, 78, 77, 74, 81, 85, 76, 86, 86, 88, 84, 87, 83, 73, 65, 78, 94, 79, 99, 92, 91, 87, 99, 83, 104, 78, 89, 104, 87, 98, 79, 100, 77, 94, 87, 95, 84, 81, 89, 100, 90, 86, 85, 87, 85, 89, 78, 86, 84, 80, 83, 107, 87, 94, 85, 95, 99, 84, 90, 79, 94, 89, 98, 76, 94, 64, 82, 90, 87, 93, 86, 76, 85, 79, 77, 69, 106, 75, 93, 84, 70, 97, 71, 84, 85, 96, 82, 83, 96, 78, 95, 86, 95, 94, 88, 73, 77, 85, 104, 75, 78, 72, 78, 85, 94, 85, 92, 95, 86, 102, 59, 80, 89, 91, 78, 85, 88, 79, 85, 78, 97, 78, 98, 86, 82, 87, 72, 91, 91, 91, 92, 81, 87, 69, 78, 94, 91, 90, 85, 84, 84, 78, 88, 69, 78, 89, 93, 105, 92, 87, 73, 96, 83, 103] payload_rows_per_dst=[5432, 5372, 5460, 5410, 5344, 5458, 5316, 5388] dup_rows_per_dst=[2826, 2628, 2758, 2878, 2768, 2990, 2828, 2680] combine_rows_per_src=[5432, 5372, 5460, 5410, 5344, 5458, 5316, 5388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43260 payload_rows_remote=37770 dup_rows=22276 dup_rows_remote=19442 dedup_ratio=0.339905 router_tokens_per_expert=[87, 88, 101, 89, 72, 86, 97, 96, 87, 80, 95, 82, 89, 78, 79, 86, 78, 74, 70, 78, 80, 86, 74, 80, 88, 84, 86, 96, 96, 90, 70, 76, 84, 92, 90, 80, 100, 73, 78, 78, 83, 78, 86, 84, 84, 77, 100, 84, 79, 85, 90, 90, 87, 89, 76, 81, 85, 97, 88, 93, 90, 96, 84, 75, 78, 73, 93, 80, 77, 92, 86, 84, 71, 85, 88, 65, 85, 91, 75, 88, 73, 82, 90, 86, 76, 71, 79, 79, 103, 90, 94, 80, 86, 80, 87, 83, 96, 98, 89, 89, 92, 75, 74, 85, 97, 87, 82, 87, 91, 83, 72, 80, 75, 78, 82, 83, 90, 85, 76, 90, 96, 79, 72, 81, 98, 91, 85, 86, 75, 96, 91, 72, 103, 78, 106, 83, 102, 90, 85, 92, 102, 78, 84, 77, 89, 89, 89, 97, 81, 92, 85, 66, 84, 85, 88, 68, 82, 93, 99, 64, 90, 71, 71, 93, 79, 90, 87, 70, 85, 78, 93, 80, 77, 115, 80, 79, 95, 76, 89, 99, 90, 68, 67, 82, 66, 92, 89, 97, 88, 85, 95, 91, 75, 75, 81, 106, 83, 100, 96, 78, 89, 80, 92, 68, 96, 93, 95, 92, 88, 91, 78, 85, 98, 100, 80, 88, 81, 80, 91, 68, 76, 87, 87, 98, 74, 75, 89, 79, 75, 77, 102, 85, 75, 69, 88, 80, 77, 80, 102, 78, 88, 85, 85, 74, 85, 100, 81, 104, 84, 75, 89, 75, 96, 98, 82, 90, 87, 80, 68, 78, 80, 87, 84, 82, 96, 76, 84, 89, 71, 95, 69, 80, 89, 78, 95, 84, 87, 72, 87, 69, 82, 88, 94, 107, 95, 70, 74, 85, 90, 81, 92, 87, 103, 86, 83, 77, 99, 97, 88, 87, 85, 78, 82, 73, 99, 89, 73, 87, 83, 75, 88, 76, 71, 87, 80, 86, 82, 88, 78, 107, 76, 79, 96, 84, 82, 77, 93, 88, 84, 83, 98, 80, 83, 91, 103, 83, 76, 104, 91, 86, 89, 80, 90, 93, 94, 88, 77, 77, 90, 75, 87, 84, 69, 98, 89, 82, 83, 79, 111, 99, 91, 93, 90, 101, 77, 68, 87, 93, 85, 79, 117, 92, 89, 89, 102, 81, 88, 99, 90, 85, 87, 104, 84, 76] payload_rows_per_dst=[5406, 5370, 5432, 5490, 5338, 5412, 5372, 5440] dup_rows_per_dst=[2692, 2700, 2844, 2834, 2822, 2694, 2862, 2828] combine_rows_per_src=[5406, 5370, 5432, 5490, 5338, 5412, 5372, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43250 payload_rows_remote=37752 dup_rows=22286 dup_rows_remote=19472 dedup_ratio=0.340057 router_tokens_per_expert=[88, 82, 79, 76, 101, 68, 77, 94, 82, 94, 89, 76, 88, 83, 76, 92, 78, 79, 78, 75, 94, 93, 78, 75, 72, 83, 105, 84, 104, 77, 83, 70, 102, 76, 82, 81, 88, 87, 94, 88, 103, 91, 94, 80, 81, 89, 82, 88, 90, 83, 94, 102, 69, 94, 83, 87, 109, 84, 85, 79, 87, 77, 105, 96, 93, 74, 82, 85, 86, 92, 65, 77, 87, 75, 86, 73, 100, 94, 79, 94, 79, 91, 87, 81, 94, 83, 88, 78, 93, 82, 96, 93, 88, 83, 83, 80, 77, 85, 83, 79, 79, 83, 86, 98, 85, 96, 92, 74, 94, 81, 85, 73, 84, 104, 78, 93, 102, 83, 77, 78, 71, 88, 100, 99, 86, 86, 92, 94, 97, 81, 88, 64, 82, 80, 73, 73, 94, 103, 89, 106, 71, 67, 80, 73, 90, 88, 85, 79, 80, 94, 85, 77, 72, 91, 86, 79, 96, 99, 82, 67, 79, 90, 79, 112, 82, 96, 94, 79, 81, 81, 102, 76, 80, 84, 85, 73, 89, 81, 78, 86, 67, 97, 79, 69, 75, 96, 69, 86, 74, 67, 96, 69, 78, 73, 77, 94, 80, 77, 85, 84, 109, 83, 78, 76, 91, 84, 85, 85, 99, 83, 69, 74, 77, 83, 91, 82, 95, 80, 96, 74, 82, 84, 82, 86, 92, 93, 97, 79, 77, 84, 75, 86, 69, 85, 85, 86, 91, 74, 95, 87, 86, 91, 99, 70, 89, 68, 92, 94, 92, 79, 91, 99, 93, 100, 99, 86, 68, 93, 83, 90, 88, 92, 92, 81, 67, 82, 92, 70, 89, 86, 83, 72, 85, 85, 98, 85, 89, 78, 81, 62, 75, 90, 65, 85, 84, 74, 106, 84, 94, 79, 84, 96, 84, 103, 83, 83, 86, 86, 86, 97, 93, 96, 97, 76, 115, 86, 95, 89, 92, 88, 102, 86, 94, 83, 60, 83, 79, 84, 68, 79, 91, 78, 97, 113, 91, 83, 76, 84, 77, 88, 87, 80, 86, 83, 89, 93, 98, 83, 63, 92, 84, 97, 86, 95, 83, 88, 83, 96, 89, 90, 90, 80, 103, 79, 81, 60, 84, 100, 89, 66, 91, 92, 85, 86, 67, 105, 100, 72, 85, 95, 90, 116, 66, 94, 85, 91, 80, 71, 88, 79, 89, 86, 94, 76] payload_rows_per_dst=[5404, 5498, 5432, 5402, 5330, 5396, 5460, 5328] dup_rows_per_dst=[2754, 2814, 2740, 2766, 2732, 2768, 2922, 2790] combine_rows_per_src=[5404, 5498, 5432, 5402, 5330, 5396, 5460, 5328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43464 payload_rows_remote=38082 dup_rows=22072 dup_rows_remote=19304 dedup_ratio=0.336792 router_tokens_per_expert=[91, 83, 92, 87, 68, 75, 83, 69, 84, 71, 92, 78, 93, 92, 78, 97, 80, 81, 90, 74, 83, 85, 78, 72, 69, 88, 82, 73, 87, 86, 91, 99, 82, 95, 120, 77, 96, 78, 107, 98, 96, 88, 75, 74, 97, 81, 81, 91, 87, 77, 93, 107, 85, 89, 83, 89, 77, 97, 74, 85, 86, 87, 78, 84, 77, 83, 99, 95, 90, 83, 83, 89, 78, 91, 65, 86, 70, 86, 89, 85, 75, 101, 83, 83, 65, 71, 90, 107, 70, 86, 88, 78, 74, 83, 97, 79, 87, 96, 73, 74, 70, 98, 66, 71, 89, 78, 90, 88, 87, 82, 98, 75, 85, 89, 88, 88, 94, 76, 91, 84, 84, 101, 101, 89, 93, 79, 69, 86, 92, 78, 82, 69, 89, 89, 74, 90, 63, 80, 90, 99, 89, 92, 95, 85, 80, 81, 86, 75, 95, 75, 117, 82, 82, 95, 104, 86, 86, 75, 104, 84, 89, 55, 78, 86, 97, 95, 103, 83, 77, 83, 64, 75, 70, 78, 82, 79, 97, 88, 87, 85, 84, 99, 114, 81, 59, 84, 73, 71, 82, 82, 90, 89, 89, 72, 90, 106, 84, 86, 99, 85, 99, 81, 90, 99, 75, 87, 73, 84, 89, 87, 68, 73, 99, 82, 80, 86, 81, 72, 94, 82, 81, 83, 70, 82, 78, 77, 87, 82, 86, 120, 80, 84, 62, 84, 93, 91, 99, 96, 77, 77, 98, 76, 89, 86, 91, 67, 76, 81, 96, 78, 91, 75, 92, 85, 106, 84, 86, 89, 84, 97, 74, 84, 92, 73, 72, 83, 82, 90, 75, 87, 99, 79, 87, 101, 86, 83, 69, 83, 78, 100, 85, 89, 79, 86, 78, 83, 88, 84, 90, 95, 85, 98, 84, 101, 93, 65, 80, 87, 89, 74, 91, 78, 81, 86, 101, 84, 87, 77, 96, 77, 78, 94, 87, 88, 75, 91, 77, 86, 97, 62, 101, 78, 82, 90, 80, 108, 82, 88, 99, 87, 80, 75, 83, 103, 86, 88, 94, 107, 79, 82, 85, 88, 96, 104, 91, 87, 85, 84, 80, 88, 103, 86, 83, 92, 84, 98, 75, 81, 99, 90, 63, 79, 84, 80, 85, 90, 108, 99, 82, 96, 89, 98, 77, 82, 75, 81, 89, 100, 87, 78, 88, 81, 68, 82] payload_rows_per_dst=[5382, 5364, 5382, 5528, 5462, 5400, 5484, 5462] dup_rows_per_dst=[2792, 2750, 2768, 2818, 2700, 2752, 2804, 2688] combine_rows_per_src=[5382, 5364, 5382, 5528, 5462, 5400, 5484, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43302 payload_rows_remote=37738 dup_rows=22234 dup_rows_remote=19378 dedup_ratio=0.339264 router_tokens_per_expert=[88, 90, 86, 92, 83, 95, 95, 90, 92, 73, 75, 84, 68, 90, 58, 90, 90, 76, 101, 77, 69, 87, 93, 93, 74, 67, 75, 102, 93, 87, 90, 83, 73, 91, 85, 89, 79, 94, 90, 73, 77, 87, 60, 77, 87, 84, 92, 79, 80, 93, 93, 88, 89, 90, 101, 86, 90, 86, 88, 90, 60, 87, 79, 86, 77, 68, 82, 91, 83, 80, 75, 90, 82, 79, 85, 82, 76, 88, 79, 91, 83, 83, 78, 80, 93, 93, 101, 81, 70, 111, 105, 82, 87, 64, 82, 70, 89, 91, 83, 82, 90, 80, 84, 80, 99, 87, 71, 89, 89, 67, 87, 85, 90, 82, 86, 84, 84, 91, 99, 87, 69, 90, 103, 81, 113, 97, 79, 115, 83, 78, 85, 91, 74, 86, 86, 92, 89, 74, 81, 79, 88, 81, 80, 82, 85, 79, 89, 98, 79, 96, 84, 75, 76, 86, 82, 97, 82, 100, 83, 72, 94, 84, 93, 92, 104, 77, 90, 66, 84, 89, 91, 79, 66, 80, 79, 80, 89, 79, 87, 94, 91, 87, 88, 85, 72, 75, 80, 89, 82, 76, 90, 90, 74, 91, 71, 77, 113, 94, 94, 79, 74, 77, 79, 73, 70, 80, 83, 87, 88, 91, 98, 91, 87, 85, 80, 78, 87, 71, 84, 66, 91, 89, 75, 88, 99, 89, 82, 69, 73, 72, 95, 91, 92, 73, 88, 89, 91, 78, 76, 88, 97, 97, 102, 84, 84, 90, 89, 91, 82, 81, 75, 82, 84, 84, 88, 76, 93, 76, 82, 95, 88, 85, 81, 100, 83, 95, 97, 95, 87, 89, 103, 90, 75, 91, 87, 82, 85, 75, 93, 86, 95, 91, 94, 75, 87, 79, 91, 90, 93, 97, 85, 91, 80, 69, 69, 78, 77, 78, 86, 104, 82, 86, 94, 74, 77, 82, 82, 87, 84, 90, 98, 90, 89, 87, 85, 82, 85, 86, 85, 87, 89, 72, 93, 70, 79, 84, 76, 84, 108, 100, 84, 99, 87, 79, 83, 110, 83, 109, 86, 94, 78, 89, 85, 82, 86, 88, 74, 99, 92, 78, 84, 83, 77, 94, 75, 101, 93, 103, 85, 95, 95, 80, 89, 83, 91, 83, 90, 77, 79, 86, 76, 66, 107, 91, 87, 86, 93, 75, 104, 78, 79, 100, 82, 74] payload_rows_per_dst=[5260, 5420, 5450, 5372, 5358, 5464, 5414, 5564] dup_rows_per_dst=[2694, 2694, 2814, 2758, 2662, 2928, 2828, 2856] combine_rows_per_src=[5260, 5420, 5450, 5372, 5358, 5464, 5414, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43394 payload_rows_remote=37944 dup_rows=22142 dup_rows_remote=19364 dedup_ratio=0.337860 router_tokens_per_expert=[87, 94, 91, 74, 85, 88, 69, 89, 76, 104, 83, 69, 96, 88, 73, 90, 94, 95, 87, 100, 103, 84, 92, 79, 81, 83, 90, 101, 75, 93, 82, 76, 88, 96, 99, 71, 74, 82, 86, 90, 98, 94, 64, 84, 106, 92, 74, 94, 82, 79, 73, 100, 104, 84, 92, 82, 77, 79, 90, 91, 70, 97, 90, 99, 91, 95, 99, 68, 75, 85, 97, 78, 84, 85, 75, 81, 84, 95, 71, 90, 89, 71, 82, 85, 79, 89, 102, 94, 87, 71, 81, 74, 77, 101, 79, 81, 87, 83, 75, 90, 79, 90, 93, 84, 81, 81, 100, 85, 100, 87, 89, 98, 102, 81, 80, 79, 77, 88, 77, 77, 82, 95, 80, 84, 76, 85, 93, 93, 81, 90, 100, 74, 81, 81, 91, 80, 103, 77, 85, 83, 91, 77, 81, 85, 89, 90, 82, 79, 83, 81, 87, 76, 77, 83, 64, 87, 88, 92, 77, 88, 67, 88, 78, 88, 88, 86, 92, 53, 95, 64, 73, 78, 80, 86, 87, 86, 80, 89, 79, 80, 86, 84, 95, 82, 79, 94, 92, 69, 75, 91, 94, 77, 77, 92, 87, 78, 88, 82, 88, 86, 106, 88, 77, 93, 97, 83, 70, 95, 70, 68, 85, 79, 79, 94, 85, 82, 86, 97, 87, 89, 101, 68, 83, 102, 95, 88, 76, 96, 87, 97, 86, 84, 91, 79, 91, 95, 88, 87, 84, 74, 84, 69, 78, 89, 88, 79, 72, 79, 93, 96, 94, 82, 85, 88, 82, 80, 91, 71, 81, 87, 85, 82, 89, 87, 71, 88, 74, 81, 102, 91, 80, 78, 78, 70, 85, 76, 92, 67, 92, 92, 86, 90, 79, 119, 83, 82, 82, 94, 89, 77, 81, 89, 79, 80, 85, 98, 81, 84, 78, 84, 75, 83, 74, 90, 83, 95, 98, 67, 99, 87, 92, 94, 78, 75, 104, 82, 87, 88, 80, 82, 91, 64, 87, 84, 87, 91, 87, 82, 91, 86, 89, 86, 97, 88, 95, 91, 85, 68, 78, 103, 94, 101, 84, 94, 78, 80, 75, 100, 97, 81, 95, 102, 81, 110, 74, 100, 94, 92, 86, 105, 111, 71, 90, 94, 86, 79, 85, 87, 69, 80, 74, 95, 83, 75, 84, 58, 96, 64, 88, 93, 90, 70, 82, 94] payload_rows_per_dst=[5512, 5462, 5428, 5330, 5454, 5320, 5450, 5438] dup_rows_per_dst=[2814, 2706, 2794, 2586, 2826, 2766, 2778, 2872] combine_rows_per_src=[5512, 5462, 5428, 5330, 5454, 5320, 5450, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542416896 dispatch_meta_bytes=229696 combine_payload_bytes=542416896 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542646592 ingress_bytes=718577664 link_bytes=718577664 token_bytes=1085063488 weight_movement_bytes=176160768 total_nvlink_bytes=1261224256 weight_to_token_ratio=0.162351 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542273536 dispatch_meta_bytes=230232 combine_payload_bytes=542273536 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=542503768 ingress_bytes=894595072 link_bytes=894595072 token_bytes=1084777304 weight_movement_bytes=352321536 total_nvlink_bytes=1437098840 weight_to_token_ratio=0.324787 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.190341 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514038471e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43216 payload_rows_remote=37686 dup_rows=22320 dup_rows_remote=19414 dedup_ratio=0.340576 router_tokens_per_expert=[93, 80, 74, 94, 96, 91, 106, 82, 77, 85, 79, 78, 93, 96, 96, 63, 98, 73, 95, 91, 80, 97, 73, 89, 81, 85, 93, 104, 102, 84, 98, 77, 88, 111, 94, 82, 83, 80, 84, 92, 90, 100, 84, 56, 92, 85, 85, 89, 80, 64, 87, 96, 71, 108, 102, 99, 87, 88, 81, 93, 86, 93, 78, 90, 82, 74, 98, 97, 92, 71, 90, 79, 80, 83, 83, 82, 83, 96, 91, 83, 88, 77, 82, 79, 84, 60, 93, 78, 100, 85, 109, 75, 81, 86, 91, 93, 78, 83, 85, 65, 71, 94, 90, 87, 86, 82, 87, 93, 74, 86, 101, 78, 62, 81, 78, 74, 99, 83, 102, 84, 74, 97, 88, 69, 83, 96, 73, 105, 73, 96, 76, 65, 70, 78, 79, 72, 88, 66, 82, 79, 70, 87, 74, 78, 91, 76, 93, 94, 84, 77, 83, 89, 74, 93, 68, 116, 77, 69, 75, 81, 103, 85, 83, 89, 88, 84, 94, 76, 82, 81, 61, 73, 83, 96, 81, 76, 84, 82, 115, 84, 68, 104, 81, 98, 79, 86, 83, 75, 85, 69, 91, 97, 89, 105, 95, 87, 83, 89, 80, 72, 96, 94, 101, 101, 99, 85, 90, 96, 91, 74, 76, 79, 86, 82, 92, 83, 76, 92, 83, 74, 97, 101, 85, 79, 90, 99, 92, 87, 87, 94, 102, 83, 82, 69, 92, 88, 91, 94, 81, 78, 93, 88, 88, 90, 86, 79, 102, 87, 102, 86, 89, 88, 88, 87, 75, 97, 77, 85, 81, 74, 76, 86, 82, 90, 67, 84, 88, 96, 85, 83, 98, 81, 93, 86, 85, 95, 88, 76, 97, 97, 82, 84, 73, 87, 77, 75, 93, 79, 86, 93, 80, 78, 94, 72, 71, 74, 94, 61, 83, 95, 96, 80, 75, 85, 85, 77, 105, 81, 96, 81, 75, 88, 105, 96, 70, 89, 93, 86, 90, 93, 77, 80, 83, 90, 79, 93, 80, 82, 79, 87, 93, 78, 104, 87, 86, 84, 78, 87, 93, 86, 82, 86, 82, 79, 84, 78, 96, 93, 88, 78, 106, 68, 84, 75, 91, 88, 81, 89, 93, 85, 80, 92, 79, 85, 94, 80, 91, 81, 85, 86, 88, 71, 59, 74, 81, 83, 88, 91, 93, 87, 71, 88, 73, 80] payload_rows_per_dst=[5530, 5446, 5294, 5436, 5536, 5480, 5234, 5260] dup_rows_per_dst=[2906, 2968, 2750, 2850, 3056, 2730, 2434, 2626] combine_rows_per_src=[5530, 5446, 5294, 5436, 5536, 5480, 5234, 5260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541470720 dispatch_meta_bytes=228848 combine_payload_bytes=541470720 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541699568 ingress_bytes=717631488 link_bytes=717631488 token_bytes=1083170288 weight_movement_bytes=176160768 total_nvlink_bytes=1259331056 weight_to_token_ratio=0.162634 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.027655168e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541011968 dispatch_meta_bytes=228464 combine_payload_bytes=541011968 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=541240432 ingress_bytes=893333504 link_bytes=893333504 token_bytes=1082252400 weight_movement_bytes=352321536 total_nvlink_bytes=1434573936 weight_to_token_ratio=0.325545 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.503442090e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541212672 dispatch_meta_bytes=228896 combine_payload_bytes=541212672 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=541441568 ingress_bytes=717373440 link_bytes=717373440 token_bytes=1082654240 weight_movement_bytes=176160768 total_nvlink_bytes=1258815008 weight_to_token_ratio=0.162712 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.025487726e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=545943552 dispatch_meta_bytes=229544 combine_payload_bytes=545943552 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=546173096 ingress_bytes=722104320 link_bytes=722104320 token_bytes=1092116648 weight_movement_bytes=176160768 total_nvlink_bytes=1268277416 weight_to_token_ratio=0.161302 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=896 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.065224156e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1085063488 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542646592 ingress_bytes=718577664 link_bytes=718577664 total_nvlink_bytes=1261224256 weight_to_token_ratio=0.162351 prefetch_share_of_ingress=0.245152 dedup_ratio=0.341125 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.035602454e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084777304 prefetch_bytes=0 reduce_bytes=352321536 weight_movement_bytes=352321536 egress_bytes=542503768 ingress_bytes=894595072 link_bytes=894595072 total_nvlink_bytes=1437098840 weight_to_token_ratio=0.324787 prefetch_share_of_ingress=0.000000 dedup_ratio=0.342133 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=2 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514038471e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083170288 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541699568 ingress_bytes=717631488 link_bytes=717631488 total_nvlink_bytes=1259331056 weight_to_token_ratio=0.162634 prefetch_share_of_ingress=0.245475 dedup_ratio=0.339905 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.027655168e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543965184 dispatch_meta_bytes=229232 combine_payload_bytes=543965184 prefetch_bytes=176160768 reduce_bytes=528482304 egress_bytes=544194416 ingress_bytes=1248608256 link_bytes=1248608256 token_bytes=1088159600 weight_movement_bytes=704643072 total_nvlink_bytes=1792802672 weight_to_token_ratio=0.647555 gemm_rows_defined=36736 gemm_rows_charged=45056 gemm_tail_pad_rows=8320 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.184659 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.048752756e-04 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1082252400 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=541240432 ingress_bytes=893333504 link_bytes=893333504 total_nvlink_bytes=1434573936 weight_to_token_ratio=0.325545 prefetch_share_of_ingress=0.197195 dedup_ratio=0.339264 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.503442090e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1092116648 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=546173096 ingress_bytes=722104320 link_bytes=722104320 total_nvlink_bytes=1268277416 weight_to_token_ratio=0.161302 prefetch_share_of_ingress=0.243955 dedup_ratio=0.336792 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.065224156e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1082654240 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=541441568 ingress_bytes=717373440 link_bytes=717373440 total_nvlink_bytes=1258815008 weight_to_token_ratio=0.162712 prefetch_share_of_ingress=0.245564 dedup_ratio=0.340057 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.025487726e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540266496 dispatch_meta_bytes=228400 combine_payload_bytes=540266496 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=540494896 ingress_bytes=892588032 link_bytes=892588032 token_bytes=1080761392 weight_movement_bytes=352321536 total_nvlink_bytes=1433082928 weight_to_token_ratio=0.325994 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.497180592e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088159600 prefetch_bytes=176160768 reduce_bytes=528482304 weight_movement_bytes=704643072 egress_bytes=544194416 ingress_bytes=1248608256 link_bytes=1248608256 total_nvlink_bytes=1792802672 weight_to_token_ratio=0.647555 prefetch_share_of_ingress=0.141086 dedup_ratio=0.337860 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=3 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.184659 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.048752756e-04 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1080761392 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=540494896 ingress_bytes=892588032 link_bytes=892588032 total_nvlink_bytes=1433082928 weight_to_token_ratio=0.325994 prefetch_share_of_ingress=0.197360 dedup_ratio=0.340576 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.497180592e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:56] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 120.2 | throughput per GPU (TFLOP/s/GPU): 294.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.082739E+01 | loss scale: 1.0 | grad norm: 8.191 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43040 payload_rows_remote=37624 dup_rows=22496 dup_rows_remote=19726 dedup_ratio=0.343262 router_tokens_per_expert=[79, 84, 95, 94, 87, 81, 89, 90, 91, 87, 85, 88, 92, 88, 78, 100, 80, 75, 84, 93, 76, 96, 88, 71, 82, 91, 75, 79, 74, 82, 80, 86, 68, 74, 76, 85, 85, 108, 72, 73, 79, 101, 85, 92, 78, 78, 81, 93, 66, 90, 74, 82, 100, 70, 90, 82, 86, 90, 86, 92, 82, 65, 90, 88, 90, 89, 81, 89, 81, 90, 90, 80, 77, 91, 74, 75, 83, 85, 85, 82, 96, 94, 84, 82, 86, 68, 97, 92, 76, 76, 73, 92, 96, 112, 94, 80, 83, 93, 90, 78, 92, 80, 83, 82, 58, 87, 100, 78, 94, 89, 88, 73, 87, 92, 108, 90, 89, 84, 83, 88, 99, 86, 79, 70, 72, 82, 77, 81, 85, 82, 95, 85, 93, 81, 83, 82, 92, 91, 71, 92, 91, 95, 73, 78, 77, 84, 87, 78, 89, 80, 96, 82, 80, 84, 69, 94, 111, 82, 92, 100, 88, 89, 80, 79, 88, 94, 80, 88, 108, 96, 78, 91, 90, 68, 102, 103, 82, 89, 92, 92, 70, 80, 88, 74, 95, 92, 64, 84, 97, 71, 97, 80, 90, 86, 91, 85, 86, 73, 96, 80, 77, 84, 92, 80, 82, 78, 79, 91, 74, 96, 83, 82, 86, 84, 84, 75, 94, 86, 69, 101, 78, 103, 83, 82, 99, 105, 77, 67, 70, 85, 94, 90, 78, 74, 94, 101, 101, 89, 74, 85, 83, 86, 103, 85, 85, 90, 94, 98, 74, 84, 91, 78, 93, 86, 86, 79, 74, 90, 90, 73, 90, 70, 80, 95, 96, 88, 85, 78, 92, 82, 86, 87, 76, 105, 75, 80, 97, 70, 87, 95, 91, 87, 74, 97, 81, 98, 87, 70, 83, 81, 80, 81, 90, 99, 88, 76, 88, 84, 89, 89, 63, 82, 74, 91, 85, 95, 74, 84, 95, 87, 67, 83, 85, 106, 97, 90, 97, 92, 82, 70, 78, 94, 94, 85, 96, 86, 71, 91, 96, 81, 69, 84, 88, 80, 100, 82, 94, 79, 91, 79, 92, 86, 85, 80, 79, 76, 95, 68, 100, 90, 70, 84, 76, 86, 77, 84, 92, 85, 75, 90, 88, 73, 80, 78, 82, 85, 90, 86, 74, 95, 83, 74, 88, 97, 90, 90, 93, 103, 94, 87, 103, 71, 92, 84] payload_rows_per_dst=[5322, 5432, 5278, 5426, 5416, 5384, 5416, 5366] dup_rows_per_dst=[2774, 2836, 2890, 2882, 2770, 2736, 2788, 2820] combine_rows_per_src=[5322, 5432, 5278, 5426, 5416, 5384, 5416, 5366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43164 payload_rows_remote=37722 dup_rows=22372 dup_rows_remote=19544 dedup_ratio=0.341370 router_tokens_per_expert=[79, 94, 94, 72, 86, 78, 81, 80, 71, 96, 105, 71, 103, 97, 94, 82, 76, 93, 98, 98, 90, 85, 69, 88, 92, 83, 72, 95, 80, 86, 90, 82, 65, 93, 96, 85, 88, 87, 95, 91, 92, 87, 68, 101, 68, 87, 96, 72, 82, 73, 77, 77, 103, 92, 91, 77, 59, 81, 77, 81, 89, 70, 99, 81, 96, 89, 80, 67, 86, 85, 86, 78, 80, 91, 90, 96, 93, 83, 81, 83, 103, 71, 83, 87, 97, 101, 77, 98, 79, 100, 81, 71, 91, 73, 90, 108, 71, 78, 74, 95, 96, 71, 72, 79, 74, 88, 80, 102, 76, 76, 95, 85, 92, 87, 70, 98, 78, 84, 88, 94, 72, 87, 89, 103, 89, 93, 97, 89, 93, 73, 89, 92, 84, 87, 81, 76, 88, 93, 86, 108, 75, 74, 88, 80, 83, 77, 82, 75, 82, 85, 80, 83, 93, 98, 83, 90, 86, 101, 78, 85, 96, 80, 85, 88, 92, 94, 72, 96, 80, 88, 84, 94, 89, 87, 85, 86, 82, 81, 100, 101, 97, 71, 81, 81, 75, 102, 90, 73, 81, 93, 77, 95, 78, 84, 90, 77, 72, 88, 87, 82, 91, 88, 82, 83, 67, 86, 81, 78, 76, 102, 88, 72, 86, 71, 81, 79, 90, 86, 88, 80, 84, 87, 84, 83, 93, 88, 83, 92, 78, 75, 83, 83, 90, 85, 85, 82, 97, 70, 68, 89, 83, 85, 92, 91, 95, 85, 86, 80, 69, 83, 79, 87, 81, 84, 74, 95, 87, 93, 88, 101, 100, 72, 86, 102, 87, 89, 94, 108, 78, 81, 75, 101, 84, 88, 73, 71, 89, 92, 93, 78, 72, 81, 78, 74, 92, 80, 89, 88, 94, 75, 109, 78, 92, 91, 100, 96, 85, 96, 86, 71, 90, 70, 86, 88, 81, 80, 87, 83, 100, 71, 87, 80, 82, 86, 106, 93, 95, 71, 83, 83, 85, 92, 73, 63, 87, 78, 71, 84, 91, 81, 100, 70, 78, 82, 96, 89, 95, 83, 85, 89, 85, 76, 96, 85, 76, 90, 93, 80, 75, 86, 66, 87, 101, 80, 91, 85, 110, 73, 89, 75, 88, 72, 83, 77, 92, 86, 89, 81, 92, 90, 66, 78, 90, 80, 115, 95, 94, 93, 89, 90, 90, 89, 82, 76] payload_rows_per_dst=[5364, 5410, 5396, 5372, 5318, 5436, 5426, 5442] dup_rows_per_dst=[2688, 2756, 2782, 2888, 2666, 3000, 2764, 2828] combine_rows_per_src=[5364, 5410, 5396, 5372, 5318, 5436, 5426, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43362 payload_rows_remote=38014 dup_rows=22174 dup_rows_remote=19372 dedup_ratio=0.338348 router_tokens_per_expert=[67, 86, 89, 76, 74, 88, 90, 90, 88, 80, 92, 93, 80, 84, 78, 96, 79, 95, 91, 79, 102, 89, 89, 82, 70, 88, 79, 84, 73, 85, 94, 76, 88, 87, 72, 88, 81, 72, 82, 79, 79, 63, 93, 85, 87, 81, 78, 95, 84, 71, 101, 86, 89, 88, 96, 97, 94, 84, 85, 61, 74, 76, 70, 95, 84, 87, 90, 99, 86, 78, 82, 64, 99, 82, 81, 81, 83, 84, 78, 75, 93, 83, 71, 75, 103, 72, 93, 85, 75, 86, 70, 89, 61, 80, 80, 92, 86, 80, 100, 105, 86, 77, 94, 81, 77, 82, 90, 82, 89, 83, 80, 91, 89, 92, 95, 79, 85, 84, 96, 77, 92, 93, 82, 81, 83, 85, 85, 101, 86, 82, 82, 83, 90, 70, 85, 81, 77, 91, 100, 93, 80, 85, 81, 77, 74, 69, 100, 95, 84, 79, 88, 84, 99, 81, 83, 85, 90, 82, 83, 79, 107, 78, 89, 82, 86, 81, 68, 103, 86, 96, 87, 80, 97, 84, 80, 90, 85, 77, 82, 101, 90, 72, 83, 81, 87, 95, 96, 83, 106, 86, 91, 95, 90, 101, 72, 93, 87, 81, 87, 76, 93, 73, 91, 90, 76, 99, 91, 72, 92, 88, 95, 92, 89, 94, 73, 85, 96, 78, 94, 84, 100, 92, 92, 93, 83, 66, 91, 100, 83, 81, 94, 99, 98, 82, 70, 87, 99, 80, 92, 83, 74, 83, 83, 73, 93, 86, 86, 91, 94, 86, 76, 74, 77, 81, 87, 103, 89, 82, 80, 100, 67, 73, 96, 84, 93, 91, 86, 93, 82, 80, 89, 79, 78, 82, 85, 80, 73, 89, 85, 94, 78, 101, 88, 80, 91, 85, 94, 109, 96, 65, 72, 96, 82, 99, 83, 74, 90, 96, 82, 93, 66, 92, 85, 84, 65, 80, 87, 71, 76, 90, 93, 72, 83, 99, 87, 79, 68, 89, 92, 96, 83, 91, 83, 78, 103, 74, 89, 69, 105, 96, 56, 101, 80, 97, 82, 70, 89, 91, 96, 78, 86, 74, 91, 76, 83, 85, 85, 81, 90, 90, 91, 87, 81, 75, 84, 85, 75, 86, 95, 102, 107, 82, 84, 86, 78, 86, 86, 82, 86, 66, 88, 88, 88, 86, 96, 99, 95, 91, 74, 85, 86, 90, 92, 80] payload_rows_per_dst=[5366, 5348, 5460, 5436, 5536, 5372, 5374, 5470] dup_rows_per_dst=[2666, 2802, 2790, 2882, 2994, 2668, 2568, 2804] combine_rows_per_src=[5366, 5348, 5460, 5436, 5536, 5372, 5374, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43230 payload_rows_remote=37814 dup_rows=22306 dup_rows_remote=19472 dedup_ratio=0.340363 router_tokens_per_expert=[77, 85, 88, 104, 81, 76, 80, 75, 87, 73, 87, 83, 78, 94, 92, 109, 94, 97, 104, 94, 88, 62, 79, 88, 79, 95, 89, 66, 88, 101, 75, 100, 94, 74, 97, 88, 94, 89, 80, 107, 85, 77, 92, 77, 85, 79, 80, 78, 79, 86, 86, 73, 71, 96, 89, 75, 100, 81, 95, 93, 85, 74, 91, 89, 80, 95, 91, 91, 86, 81, 92, 83, 101, 90, 75, 76, 95, 93, 86, 82, 79, 82, 72, 85, 93, 91, 95, 67, 87, 92, 93, 78, 81, 80, 88, 80, 84, 73, 82, 77, 91, 80, 101, 84, 86, 90, 97, 83, 79, 76, 105, 76, 90, 86, 113, 83, 95, 81, 93, 78, 85, 94, 84, 84, 88, 78, 87, 89, 80, 71, 75, 78, 72, 80, 83, 84, 76, 91, 109, 99, 95, 92, 91, 77, 87, 91, 66, 79, 74, 60, 88, 64, 92, 92, 94, 82, 87, 67, 77, 93, 94, 79, 104, 82, 88, 89, 87, 104, 82, 86, 75, 93, 90, 99, 79, 95, 94, 90, 85, 76, 99, 75, 74, 82, 97, 83, 94, 83, 101, 90, 83, 84, 88, 71, 75, 96, 77, 87, 95, 79, 90, 79, 74, 85, 86, 108, 96, 78, 77, 96, 83, 75, 77, 94, 101, 86, 87, 92, 91, 79, 84, 114, 87, 96, 63, 99, 82, 71, 71, 85, 67, 89, 81, 79, 93, 88, 95, 72, 82, 92, 117, 80, 95, 66, 75, 85, 81, 71, 79, 73, 93, 92, 75, 78, 70, 90, 96, 92, 78, 77, 93, 86, 91, 82, 70, 83, 81, 92, 99, 84, 88, 77, 108, 90, 78, 70, 87, 77, 110, 70, 80, 79, 94, 80, 67, 75, 86, 88, 80, 78, 88, 86, 95, 95, 77, 87, 83, 97, 77, 78, 72, 80, 83, 96, 65, 84, 83, 81, 98, 96, 72, 86, 75, 88, 87, 87, 70, 93, 92, 96, 75, 100, 71, 99, 96, 86, 77, 92, 87, 84, 79, 79, 91, 74, 82, 76, 91, 89, 89, 75, 75, 89, 87, 69, 76, 92, 80, 94, 102, 86, 76, 79, 80, 77, 87, 88, 81, 105, 89, 94, 66, 84, 71, 89, 86, 104, 73, 82, 77, 87, 88, 95, 99, 98, 94, 102, 88, 91, 82, 81, 80, 62, 88, 97] payload_rows_per_dst=[5484, 5512, 5416, 5388, 5400, 5254, 5324, 5452] dup_rows_per_dst=[2804, 2884, 2834, 2830, 2784, 2612, 2782, 2776] combine_rows_per_src=[5484, 5512, 5416, 5388, 5400, 5254, 5324, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43268 payload_rows_remote=37828 dup_rows=22268 dup_rows_remote=19354 dedup_ratio=0.339783 router_tokens_per_expert=[74, 100, 79, 89, 70, 78, 73, 94, 83, 80, 84, 82, 99, 83, 81, 83, 82, 91, 102, 94, 77, 77, 94, 98, 61, 78, 89, 84, 86, 82, 79, 89, 73, 79, 77, 92, 87, 90, 94, 102, 94, 99, 87, 86, 84, 87, 87, 78, 92, 85, 80, 80, 75, 88, 65, 86, 89, 71, 83, 91, 72, 84, 80, 83, 79, 97, 83, 89, 75, 84, 85, 98, 81, 78, 88, 78, 86, 88, 93, 70, 87, 76, 77, 88, 75, 76, 86, 83, 70, 78, 79, 89, 85, 87, 90, 85, 67, 90, 100, 92, 82, 82, 102, 96, 91, 87, 99, 78, 86, 76, 95, 74, 82, 77, 80, 80, 80, 80, 85, 92, 76, 102, 86, 90, 88, 88, 74, 81, 67, 86, 71, 80, 90, 83, 82, 70, 80, 79, 79, 77, 80, 108, 87, 89, 77, 83, 82, 99, 81, 78, 95, 95, 83, 79, 87, 83, 94, 92, 98, 88, 97, 81, 70, 79, 93, 74, 96, 81, 90, 82, 85, 86, 95, 85, 85, 91, 68, 81, 93, 95, 94, 70, 100, 89, 72, 79, 98, 76, 110, 95, 72, 88, 81, 85, 92, 87, 70, 91, 91, 89, 77, 76, 100, 79, 83, 85, 90, 80, 91, 70, 101, 95, 83, 86, 83, 107, 95, 86, 87, 87, 81, 104, 94, 99, 87, 97, 74, 81, 84, 86, 96, 87, 85, 73, 105, 89, 70, 75, 98, 87, 96, 103, 102, 76, 76, 93, 89, 88, 88, 103, 65, 87, 89, 97, 90, 70, 101, 87, 89, 93, 85, 87, 87, 90, 94, 75, 90, 86, 85, 85, 73, 78, 80, 95, 71, 70, 105, 79, 84, 88, 82, 68, 80, 103, 88, 91, 87, 74, 76, 85, 89, 85, 81, 92, 81, 102, 98, 92, 77, 79, 71, 85, 84, 92, 78, 76, 84, 79, 90, 75, 92, 87, 72, 82, 96, 89, 89, 91, 77, 83, 107, 85, 86, 87, 83, 81, 88, 82, 102, 89, 89, 84, 83, 84, 98, 85, 78, 97, 80, 80, 86, 77, 100, 83, 88, 90, 72, 81, 92, 100, 89, 83, 85, 74, 86, 81, 88, 62, 97, 93, 88, 77, 80, 79, 87, 90, 96, 77, 79, 91, 90, 73, 100, 93, 81, 77, 86, 86, 80, 80, 100, 65, 80, 100] payload_rows_per_dst=[5324, 5310, 5374, 5382, 5508, 5440, 5446, 5484] dup_rows_per_dst=[2788, 2624, 2718, 2906, 2850, 2914, 2798, 2670] combine_rows_per_src=[5324, 5310, 5374, 5382, 5508, 5440, 5446, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43382 payload_rows_remote=37972 dup_rows=22154 dup_rows_remote=19420 dedup_ratio=0.338043 router_tokens_per_expert=[76, 106, 90, 81, 77, 81, 93, 96, 86, 76, 99, 83, 94, 77, 83, 94, 108, 81, 82, 89, 77, 70, 87, 81, 94, 91, 92, 82, 84, 86, 104, 89, 80, 79, 78, 82, 86, 86, 104, 84, 74, 90, 81, 73, 65, 91, 89, 80, 93, 62, 88, 83, 93, 82, 83, 89, 83, 105, 72, 102, 76, 89, 78, 84, 73, 87, 88, 82, 94, 90, 73, 69, 92, 76, 77, 72, 77, 71, 90, 62, 72, 83, 76, 102, 105, 86, 99, 83, 81, 85, 87, 90, 74, 77, 94, 97, 81, 81, 84, 73, 93, 84, 88, 76, 84, 96, 102, 82, 81, 100, 65, 105, 80, 87, 80, 95, 103, 89, 97, 77, 102, 84, 95, 115, 86, 82, 90, 92, 89, 77, 92, 74, 76, 74, 85, 67, 77, 86, 85, 64, 65, 86, 99, 92, 95, 81, 67, 83, 66, 79, 96, 112, 81, 79, 83, 83, 87, 78, 76, 91, 97, 78, 79, 88, 82, 95, 84, 90, 84, 85, 87, 91, 105, 89, 81, 86, 74, 71, 89, 88, 88, 96, 91, 76, 84, 81, 62, 93, 84, 89, 89, 79, 95, 82, 92, 82, 92, 79, 71, 70, 85, 81, 88, 83, 87, 86, 97, 61, 91, 78, 88, 86, 87, 77, 89, 77, 73, 85, 95, 84, 84, 89, 95, 90, 75, 78, 90, 91, 89, 71, 87, 80, 85, 75, 87, 63, 92, 74, 77, 95, 79, 90, 81, 78, 85, 88, 97, 68, 91, 96, 89, 101, 93, 83, 104, 77, 70, 90, 84, 97, 95, 69, 92, 81, 83, 94, 95, 103, 99, 81, 69, 61, 102, 98, 78, 80, 76, 78, 89, 82, 83, 82, 81, 92, 80, 79, 97, 96, 86, 83, 100, 87, 100, 96, 97, 83, 81, 68, 83, 81, 83, 83, 97, 90, 86, 114, 78, 82, 91, 86, 73, 82, 88, 81, 96, 96, 90, 81, 87, 84, 101, 83, 97, 84, 93, 104, 94, 78, 86, 85, 85, 89, 92, 81, 80, 80, 101, 83, 77, 85, 83, 84, 90, 80, 78, 100, 104, 95, 92, 87, 94, 76, 71, 86, 106, 88, 98, 86, 79, 78, 89, 78, 75, 82, 74, 80, 71, 81, 90, 85, 81, 96, 87, 86, 82, 98, 89, 104, 76, 70, 94, 76, 65, 83] payload_rows_per_dst=[5402, 5478, 5462, 5410, 5376, 5330, 5482, 5442] dup_rows_per_dst=[2820, 2736, 2772, 2734, 2640, 2780, 2928, 2744] combine_rows_per_src=[5402, 5478, 5462, 5410, 5376, 5330, 5482, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43344 payload_rows_remote=37892 dup_rows=22192 dup_rows_remote=19368 dedup_ratio=0.338623 router_tokens_per_expert=[97, 93, 102, 89, 76, 81, 90, 93, 88, 102, 96, 81, 100, 87, 87, 95, 92, 102, 94, 79, 86, 77, 85, 93, 95, 78, 88, 79, 80, 82, 79, 85, 76, 108, 102, 99, 80, 107, 83, 77, 93, 98, 105, 73, 99, 82, 90, 87, 71, 80, 101, 66, 91, 83, 78, 96, 86, 90, 95, 80, 85, 72, 82, 97, 76, 100, 85, 88, 80, 95, 79, 94, 102, 92, 100, 78, 95, 87, 75, 93, 87, 79, 89, 86, 105, 72, 77, 90, 75, 86, 84, 98, 75, 80, 76, 90, 87, 73, 99, 78, 90, 86, 97, 86, 87, 97, 86, 82, 63, 90, 87, 87, 105, 83, 89, 81, 99, 84, 87, 76, 87, 81, 78, 82, 73, 72, 84, 86, 81, 70, 84, 80, 97, 79, 83, 100, 76, 85, 71, 79, 83, 72, 87, 90, 77, 58, 74, 91, 83, 75, 77, 87, 82, 91, 75, 80, 97, 89, 77, 75, 73, 84, 77, 93, 85, 95, 81, 93, 74, 72, 85, 84, 77, 73, 92, 71, 97, 105, 75, 85, 90, 72, 75, 104, 85, 85, 86, 86, 73, 90, 94, 78, 91, 80, 84, 94, 77, 93, 93, 80, 89, 84, 86, 64, 85, 79, 87, 73, 87, 90, 89, 82, 80, 76, 76, 90, 76, 88, 72, 77, 94, 82, 81, 75, 103, 63, 90, 72, 82, 88, 90, 84, 85, 88, 102, 72, 80, 83, 79, 86, 79, 95, 101, 80, 81, 102, 83, 84, 70, 83, 84, 104, 89, 84, 96, 87, 81, 89, 99, 93, 99, 68, 85, 81, 91, 73, 86, 77, 85, 72, 82, 97, 97, 94, 87, 87, 95, 91, 91, 81, 86, 87, 83, 79, 102, 88, 90, 92, 77, 85, 93, 84, 83, 56, 94, 83, 88, 99, 78, 93, 78, 82, 95, 98, 81, 72, 70, 95, 75, 110, 83, 82, 95, 101, 67, 92, 85, 97, 89, 85, 84, 76, 91, 89, 97, 74, 92, 113, 87, 72, 85, 85, 75, 89, 87, 97, 84, 82, 90, 86, 82, 81, 89, 87, 87, 87, 78, 80, 79, 95, 74, 70, 81, 97, 104, 89, 77, 96, 82, 89, 79, 82, 90, 64, 84, 80, 95, 85, 77, 92, 75, 89, 84, 83, 84, 73, 94, 89, 75, 72, 82, 78, 74, 86] payload_rows_per_dst=[5494, 5434, 5326, 5272, 5370, 5636, 5452, 5360] dup_rows_per_dst=[2894, 2808, 2752, 2682, 2632, 2936, 2824, 2664] combine_rows_per_src=[5494, 5434, 5326, 5272, 5370, 5636, 5452, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=42872 payload_rows_remote=37476 dup_rows=22664 dup_rows_remote=19714 dedup_ratio=0.345825 router_tokens_per_expert=[72, 67, 75, 96, 91, 85, 79, 83, 82, 76, 100, 86, 92, 91, 87, 83, 65, 85, 92, 78, 89, 75, 94, 96, 79, 81, 94, 73, 82, 112, 86, 77, 95, 87, 95, 92, 96, 89, 78, 81, 83, 87, 84, 64, 76, 102, 81, 86, 90, 68, 93, 86, 83, 84, 93, 101, 70, 81, 77, 80, 81, 75, 82, 91, 82, 87, 77, 77, 76, 78, 77, 83, 98, 95, 79, 83, 88, 80, 91, 76, 88, 92, 91, 78, 93, 83, 63, 72, 87, 68, 78, 90, 78, 78, 97, 94, 69, 75, 99, 104, 90, 86, 66, 84, 73, 86, 99, 71, 89, 84, 78, 84, 88, 82, 99, 91, 74, 94, 101, 76, 81, 102, 74, 93, 60, 95, 77, 74, 89, 83, 83, 86, 95, 81, 106, 89, 94, 102, 78, 80, 90, 96, 91, 75, 80, 88, 72, 84, 101, 93, 94, 103, 90, 80, 82, 76, 93, 97, 89, 67, 80, 77, 70, 81, 91, 92, 88, 76, 84, 78, 76, 94, 84, 114, 87, 75, 75, 83, 80, 87, 80, 91, 84, 75, 91, 84, 104, 96, 94, 74, 80, 103, 73, 83, 99, 92, 84, 99, 93, 92, 78, 98, 81, 66, 91, 88, 92, 73, 85, 80, 65, 76, 85, 75, 89, 87, 69, 97, 93, 79, 73, 87, 89, 83, 99, 90, 65, 88, 84, 81, 89, 101, 72, 78, 89, 78, 86, 80, 77, 90, 70, 93, 91, 73, 87, 96, 94, 88, 81, 85, 99, 94, 77, 96, 87, 85, 100, 88, 85, 91, 77, 102, 85, 93, 92, 96, 75, 66, 81, 85, 81, 97, 105, 81, 85, 86, 95, 88, 89, 83, 69, 73, 88, 75, 68, 91, 71, 73, 78, 87, 71, 82, 81, 82, 91, 79, 111, 101, 87, 86, 73, 84, 97, 72, 80, 76, 82, 86, 75, 89, 76, 85, 100, 97, 76, 99, 88, 87, 86, 84, 105, 76, 90, 96, 97, 93, 81, 65, 94, 79, 81, 94, 84, 78, 94, 81, 84, 77, 80, 93, 91, 81, 80, 98, 82, 104, 75, 89, 79, 80, 73, 104, 107, 91, 84, 87, 101, 87, 76, 84, 90, 79, 105, 85, 87, 84, 83, 86, 92, 84, 93, 81, 79, 86, 83, 110, 95, 90, 93, 96, 80, 86, 78, 85] payload_rows_per_dst=[5396, 5372, 5396, 5272, 5396, 5330, 5376, 5334] dup_rows_per_dst=[2950, 2794, 2890, 2774, 2880, 2708, 2788, 2880] combine_rows_per_src=[5396, 5372, 5396, 5272, 5396, 5330, 5376, 5334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=539377664 dispatch_meta_bytes=229400 combine_payload_bytes=539377664 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=539607064 ingress_bytes=715538432 link_bytes=715538432 token_bytes=1078984728 weight_movement_bytes=176160768 total_nvlink_bytes=1255145496 weight_to_token_ratio=0.163265 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.010074808e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540782592 dispatch_meta_bytes=229064 combine_payload_bytes=540782592 prefetch_bytes=176160768 reduce_bytes=352321536 egress_bytes=541011656 ingress_bytes=1069264896 link_bytes=1069264896 token_bytes=1081794248 weight_movement_bytes=528482304 total_nvlink_bytes=1610276552 weight_to_token_ratio=0.488524 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.981155627e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544968704 dispatch_meta_bytes=229544 combine_payload_bytes=544968704 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=545198248 ingress_bytes=721129472 link_bytes=721129472 token_bytes=1090166952 weight_movement_bytes=176160768 total_nvlink_bytes=1266327720 weight_to_token_ratio=0.161591 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.057036043e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542101504 dispatch_meta_bytes=229144 combine_payload_bytes=542101504 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=542330648 ingress_bytes=718262272 link_bytes=718262272 token_bytes=1084432152 weight_movement_bytes=176160768 total_nvlink_bytes=1260592920 weight_to_token_ratio=0.162445 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.032953359e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542302208 dispatch_meta_bytes=228728 combine_payload_bytes=542302208 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542530936 ingress_bytes=894623744 link_bytes=894623744 token_bytes=1084833144 weight_movement_bytes=352321536 total_nvlink_bytes=1437154680 weight_to_token_ratio=0.324770 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514279298e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=544366592 dispatch_meta_bytes=229568 combine_payload_bytes=544366592 prefetch_bytes=0 reduce_bytes=352321536 egress_bytes=544596160 ingress_bytes=896688128 link_bytes=896688128 token_bytes=1088962752 weight_movement_bytes=352321536 total_nvlink_bytes=1441284288 weight_to_token_ratio=0.323539 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.193182 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.531618831e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543219712 dispatch_meta_bytes=229040 combine_payload_bytes=543219712 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=543448752 ingress_bytes=895541248 link_bytes=895541248 token_bytes=1086668464 weight_movement_bytes=352321536 total_nvlink_bytes=1438990000 weight_to_token_ratio=0.324222 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.190341 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.521985757e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1078984728 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=539607064 ingress_bytes=715538432 link_bytes=715538432 total_nvlink_bytes=1255145496 weight_to_token_ratio=0.163265 prefetch_share_of_ingress=0.246193 dedup_ratio=0.343262 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.010074808e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1081794248 prefetch_bytes=176160768 reduce_bytes=352321536 weight_movement_bytes=528482304 egress_bytes=541011656 ingress_bytes=1069264896 link_bytes=1069264896 total_nvlink_bytes=1610276552 weight_to_token_ratio=0.488524 prefetch_share_of_ingress=0.164749 dedup_ratio=0.341370 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=2 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=8.981155627e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1090166952 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=545198248 ingress_bytes=721129472 link_bytes=721129472 total_nvlink_bytes=1266327720 weight_to_token_ratio=0.161591 prefetch_share_of_ingress=0.244285 dedup_ratio=0.338348 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.057036043e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=537255936 dispatch_meta_bytes=228760 combine_payload_bytes=537255936 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=537484696 ingress_bytes=889577472 link_bytes=889577472 token_bytes=1074740632 weight_movement_bytes=352321536 total_nvlink_bytes=1427062168 weight_to_token_ratio=0.327820 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.471893773e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084432152 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=542330648 ingress_bytes=718262272 link_bytes=718262272 total_nvlink_bytes=1260592920 weight_to_token_ratio=0.162445 prefetch_share_of_ingress=0.245260 dedup_ratio=0.340363 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.032953359e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084833144 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542530936 ingress_bytes=894623744 link_bytes=894623744 total_nvlink_bytes=1437154680 weight_to_token_ratio=0.324770 prefetch_share_of_ingress=0.196910 dedup_ratio=0.339783 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514279298e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1088962752 prefetch_bytes=0 reduce_bytes=352321536 weight_movement_bytes=352321536 egress_bytes=544596160 ingress_bytes=896688128 link_bytes=896688128 total_nvlink_bytes=1441284288 weight_to_token_ratio=0.323539 prefetch_share_of_ingress=0.000000 dedup_ratio=0.338043 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=2 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.531618831e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1086668464 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=543448752 ingress_bytes=895541248 link_bytes=895541248 total_nvlink_bytes=1438990000 weight_to_token_ratio=0.324222 prefetch_share_of_ingress=0.196709 dedup_ratio=0.338623 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.521985757e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1074740632 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=537484696 ingress_bytes=889577472 link_bytes=889577472 total_nvlink_bytes=1427062168 weight_to_token_ratio=0.327820 prefetch_share_of_ingress=0.198027 dedup_ratio=0.345825 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.471893773e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:56] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 119.9 | throughput per GPU (TFLOP/s/GPU): 295.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.067843E+01 | loss scale: 1.0 | grad norm: 8.571 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43230 payload_rows_remote=37790 dup_rows=22306 dup_rows_remote=19474 dedup_ratio=0.340363 router_tokens_per_expert=[94, 67, 100, 93, 105, 83, 91, 89, 91, 71, 83, 81, 87, 84, 91, 80, 102, 86, 79, 88, 101, 94, 76, 86, 79, 63, 80, 74, 83, 98, 99, 92, 99, 98, 82, 80, 95, 82, 71, 91, 79, 84, 90, 81, 93, 71, 86, 97, 95, 77, 57, 81, 73, 80, 85, 70, 81, 98, 101, 82, 94, 80, 80, 78, 84, 83, 102, 81, 86, 101, 75, 81, 89, 85, 96, 91, 91, 86, 95, 85, 74, 81, 80, 88, 84, 86, 89, 98, 71, 84, 86, 84, 78, 97, 91, 73, 83, 91, 86, 87, 79, 79, 84, 65, 95, 72, 74, 101, 84, 104, 82, 91, 84, 79, 95, 70, 77, 95, 95, 88, 82, 76, 81, 92, 85, 89, 88, 81, 87, 95, 97, 89, 97, 93, 80, 101, 76, 83, 92, 91, 106, 70, 79, 86, 93, 99, 81, 87, 80, 68, 101, 99, 81, 83, 90, 87, 76, 76, 86, 90, 66, 99, 81, 89, 63, 94, 90, 98, 97, 83, 95, 83, 91, 78, 89, 81, 82, 73, 71, 97, 68, 97, 73, 97, 78, 95, 80, 77, 89, 75, 84, 63, 68, 85, 84, 75, 79, 65, 71, 78, 98, 86, 99, 88, 107, 69, 91, 75, 86, 99, 100, 92, 93, 65, 95, 84, 77, 110, 85, 77, 87, 108, 88, 94, 72, 100, 89, 99, 87, 87, 74, 96, 91, 81, 105, 105, 75, 78, 77, 77, 74, 78, 84, 81, 77, 86, 73, 80, 80, 81, 87, 87, 87, 84, 84, 75, 97, 95, 94, 74, 99, 90, 87, 98, 74, 86, 84, 94, 80, 74, 84, 70, 64, 67, 82, 99, 85, 84, 81, 93, 96, 81, 90, 91, 65, 94, 94, 99, 87, 84, 84, 72, 78, 94, 91, 76, 92, 92, 75, 83, 99, 90, 77, 84, 95, 95, 73, 87, 75, 96, 88, 99, 94, 95, 103, 85, 92, 75, 90, 86, 76, 77, 75, 87, 87, 70, 97, 75, 79, 100, 81, 96, 93, 80, 84, 89, 98, 75, 71, 90, 84, 69, 80, 81, 82, 72, 78, 85, 98, 83, 82, 89, 90, 73, 94, 79, 87, 74, 79, 94, 78, 81, 77, 96, 101, 81, 79, 84, 85, 88, 83, 90, 77, 87, 84, 102, 84, 78, 78, 98, 95, 83, 76, 85] payload_rows_per_dst=[5404, 5342, 5440, 5294, 5432, 5454, 5470, 5394] dup_rows_per_dst=[2894, 2792, 2832, 2812, 2870, 2718, 2708, 2680] combine_rows_per_src=[5404, 5342, 5440, 5294, 5432, 5454, 5470, 5394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43012 payload_rows_remote=37678 dup_rows=22524 dup_rows_remote=19700 dedup_ratio=0.343689 router_tokens_per_expert=[66, 79, 93, 89, 83, 104, 99, 68, 89, 82, 76, 87, 89, 84, 94, 90, 89, 79, 88, 98, 83, 89, 80, 86, 90, 89, 82, 88, 88, 79, 85, 75, 109, 75, 87, 84, 76, 91, 90, 74, 95, 86, 79, 88, 76, 95, 94, 91, 90, 82, 70, 105, 74, 78, 83, 93, 91, 88, 76, 85, 79, 92, 71, 79, 80, 93, 87, 68, 77, 94, 88, 82, 103, 77, 107, 103, 106, 83, 79, 101, 92, 109, 76, 65, 81, 68, 86, 83, 83, 77, 92, 83, 90, 81, 83, 90, 91, 68, 74, 94, 97, 85, 74, 88, 83, 87, 103, 77, 88, 99, 96, 86, 88, 80, 82, 70, 80, 90, 79, 91, 83, 68, 92, 99, 87, 78, 87, 84, 87, 77, 106, 79, 94, 85, 93, 88, 84, 76, 78, 75, 104, 82, 77, 92, 82, 87, 100, 89, 77, 86, 91, 74, 95, 80, 79, 86, 77, 76, 88, 82, 83, 90, 82, 94, 78, 80, 98, 75, 91, 77, 89, 87, 85, 100, 73, 96, 77, 77, 85, 85, 81, 86, 84, 77, 88, 66, 81, 80, 91, 79, 87, 80, 66, 87, 89, 83, 79, 87, 86, 78, 84, 86, 84, 96, 89, 75, 81, 94, 94, 85, 78, 97, 90, 94, 92, 86, 98, 89, 84, 84, 75, 85, 76, 87, 89, 81, 79, 74, 89, 80, 75, 92, 86, 82, 85, 76, 66, 87, 94, 75, 91, 74, 89, 83, 78, 76, 81, 84, 93, 61, 66, 90, 101, 75, 89, 87, 84, 85, 90, 80, 89, 76, 85, 76, 81, 92, 72, 100, 78, 87, 89, 81, 81, 99, 82, 91, 94, 81, 69, 100, 111, 91, 85, 97, 82, 94, 71, 88, 82, 91, 78, 89, 77, 87, 89, 83, 80, 79, 100, 89, 82, 96, 80, 91, 85, 93, 81, 90, 83, 78, 96, 90, 88, 72, 77, 93, 87, 94, 73, 80, 79, 93, 95, 82, 82, 89, 80, 84, 95, 92, 86, 102, 86, 78, 103, 83, 99, 76, 85, 104, 78, 77, 76, 90, 67, 88, 84, 82, 92, 75, 88, 77, 84, 88, 102, 78, 91, 92, 87, 73, 73, 93, 92, 108, 103, 77, 84, 78, 85, 79, 71, 92, 84, 86, 92, 78, 104, 84, 84, 81, 82, 95, 101, 91] payload_rows_per_dst=[5412, 5344, 5246, 5408, 5334, 5390, 5456, 5422] dup_rows_per_dst=[2848, 2800, 2766, 2852, 2824, 2768, 2828, 2838] combine_rows_per_src=[5412, 5344, 5246, 5408, 5334, 5390, 5456, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43236 payload_rows_remote=37918 dup_rows=22300 dup_rows_remote=19538 dedup_ratio=0.340271 router_tokens_per_expert=[78, 80, 93, 85, 77, 74, 84, 84, 101, 79, 67, 115, 72, 80, 73, 101, 94, 97, 88, 75, 105, 78, 79, 92, 84, 82, 88, 88, 93, 79, 88, 92, 84, 96, 75, 85, 103, 66, 92, 85, 71, 90, 88, 86, 93, 84, 76, 78, 86, 96, 76, 95, 78, 73, 67, 80, 97, 82, 98, 100, 100, 86, 102, 77, 94, 91, 89, 94, 111, 92, 81, 92, 80, 89, 88, 87, 112, 90, 93, 88, 79, 75, 82, 59, 78, 92, 83, 98, 84, 94, 83, 77, 71, 95, 87, 79, 86, 72, 91, 88, 74, 82, 87, 101, 91, 80, 97, 83, 83, 94, 71, 81, 88, 78, 88, 82, 79, 69, 98, 80, 85, 88, 76, 72, 96, 96, 69, 104, 106, 77, 95, 91, 74, 100, 78, 73, 100, 75, 95, 89, 91, 90, 75, 100, 73, 84, 79, 90, 80, 77, 66, 88, 81, 86, 65, 72, 99, 83, 87, 80, 85, 87, 88, 87, 80, 66, 76, 80, 86, 84, 94, 85, 86, 70, 91, 90, 82, 93, 87, 113, 77, 86, 86, 88, 82, 88, 79, 74, 91, 80, 69, 74, 88, 82, 87, 90, 81, 70, 64, 75, 80, 92, 91, 84, 81, 80, 86, 80, 78, 85, 69, 93, 82, 96, 85, 78, 92, 89, 75, 84, 80, 75, 67, 89, 74, 93, 75, 75, 93, 87, 80, 84, 83, 80, 100, 87, 70, 84, 83, 86, 72, 80, 86, 88, 91, 80, 80, 102, 76, 85, 100, 75, 94, 58, 91, 82, 74, 82, 96, 78, 62, 88, 84, 80, 91, 65, 75, 90, 69, 82, 82, 96, 90, 79, 82, 84, 84, 69, 68, 89, 65, 106, 83, 81, 103, 81, 66, 107, 79, 88, 90, 84, 91, 77, 91, 72, 75, 82, 78, 78, 98, 98, 93, 79, 78, 86, 100, 87, 100, 92, 90, 95, 102, 84, 92, 103, 69, 94, 87, 103, 98, 84, 93, 90, 93, 74, 88, 92, 95, 110, 100, 90, 75, 96, 82, 81, 104, 86, 87, 86, 82, 93, 95, 82, 88, 81, 97, 102, 86, 78, 80, 95, 83, 91, 86, 107, 81, 78, 87, 79, 71, 94, 96, 87, 102, 85, 70, 78, 88, 95, 83, 103, 80, 86, 103, 75, 82, 99, 92, 85, 77, 88, 85, 92] payload_rows_per_dst=[5464, 5454, 5320, 5318, 5276, 5304, 5524, 5576] dup_rows_per_dst=[2730, 2906, 2784, 2762, 2648, 2638, 2988, 2844] combine_rows_per_src=[5464, 5454, 5320, 5318, 5276, 5304, 5524, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43224 payload_rows_remote=37716 dup_rows=22312 dup_rows_remote=19532 dedup_ratio=0.340454 router_tokens_per_expert=[63, 80, 83, 102, 92, 88, 65, 94, 104, 68, 87, 102, 83, 80, 80, 87, 91, 83, 91, 88, 94, 91, 77, 89, 80, 72, 85, 75, 87, 97, 93, 84, 89, 97, 83, 74, 87, 86, 77, 85, 89, 79, 70, 75, 89, 92, 72, 93, 72, 78, 83, 91, 86, 70, 103, 97, 79, 96, 89, 83, 82, 76, 94, 73, 76, 88, 90, 70, 90, 100, 85, 94, 93, 90, 73, 78, 90, 89, 100, 74, 88, 82, 87, 88, 90, 82, 93, 101, 87, 75, 89, 86, 85, 99, 91, 89, 69, 78, 97, 71, 84, 86, 99, 86, 90, 79, 83, 78, 81, 101, 65, 82, 77, 91, 93, 84, 94, 91, 105, 86, 77, 76, 89, 88, 81, 99, 87, 99, 71, 84, 77, 87, 77, 95, 92, 80, 85, 99, 76, 78, 104, 92, 75, 92, 86, 88, 80, 83, 74, 79, 92, 82, 78, 91, 87, 74, 75, 73, 82, 84, 93, 77, 81, 91, 67, 77, 73, 89, 113, 98, 87, 89, 87, 83, 86, 80, 88, 87, 87, 96, 94, 88, 89, 84, 70, 78, 83, 70, 83, 82, 88, 102, 73, 100, 82, 69, 82, 80, 94, 73, 87, 78, 74, 82, 80, 85, 78, 87, 87, 99, 72, 92, 89, 73, 84, 83, 97, 90, 77, 77, 84, 95, 84, 82, 88, 81, 84, 91, 101, 90, 94, 87, 86, 72, 100, 72, 82, 74, 89, 90, 92, 61, 98, 73, 83, 87, 99, 89, 83, 70, 92, 94, 84, 80, 76, 85, 92, 95, 87, 79, 73, 64, 70, 97, 100, 82, 69, 62, 85, 97, 73, 106, 84, 91, 63, 99, 85, 80, 104, 92, 115, 71, 90, 79, 101, 86, 89, 74, 92, 76, 83, 98, 93, 90, 77, 110, 86, 93, 76, 63, 81, 101, 84, 83, 85, 101, 77, 97, 101, 88, 78, 80, 85, 84, 80, 81, 89, 79, 82, 78, 86, 70, 89, 87, 80, 89, 75, 89, 62, 68, 86, 85, 96, 92, 97, 84, 77, 84, 83, 95, 76, 103, 82, 82, 76, 91, 89, 96, 79, 87, 92, 81, 96, 93, 90, 88, 97, 91, 93, 86, 73, 90, 90, 90, 89, 86, 80, 86, 69, 82, 99, 86, 81, 93, 102, 102, 87, 84, 86, 90, 95, 76, 78, 77] payload_rows_per_dst=[5390, 5508, 5506, 5336, 5292, 5450, 5364, 5378] dup_rows_per_dst=[2754, 2780, 2822, 2760, 2808, 2912, 2606, 2870] combine_rows_per_src=[5390, 5508, 5506, 5336, 5292, 5450, 5364, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43226 payload_rows_remote=37786 dup_rows=22310 dup_rows_remote=19496 dedup_ratio=0.340424 router_tokens_per_expert=[74, 76, 79, 78, 89, 95, 103, 66, 87, 90, 99, 99, 93, 70, 86, 60, 83, 71, 95, 83, 89, 98, 82, 77, 87, 84, 87, 91, 84, 89, 89, 82, 80, 78, 89, 79, 90, 92, 81, 94, 84, 93, 90, 80, 72, 85, 90, 86, 74, 88, 87, 89, 87, 94, 83, 79, 76, 83, 96, 99, 96, 106, 96, 76, 82, 81, 82, 79, 87, 87, 81, 85, 82, 91, 82, 74, 88, 81, 106, 66, 80, 89, 90, 69, 82, 84, 84, 94, 75, 84, 78, 73, 87, 91, 60, 82, 102, 93, 94, 74, 92, 81, 88, 93, 88, 79, 98, 95, 96, 104, 78, 76, 102, 79, 80, 85, 85, 92, 86, 97, 82, 74, 91, 80, 94, 89, 88, 81, 82, 98, 90, 86, 103, 89, 94, 83, 82, 88, 83, 88, 81, 84, 83, 92, 77, 79, 87, 73, 73, 65, 76, 75, 77, 76, 105, 86, 84, 70, 93, 84, 76, 79, 68, 82, 78, 84, 76, 96, 92, 86, 77, 84, 68, 80, 98, 84, 107, 78, 92, 93, 88, 101, 88, 105, 87, 84, 88, 75, 92, 82, 92, 82, 82, 72, 90, 89, 60, 75, 81, 86, 91, 79, 87, 67, 94, 87, 88, 87, 89, 87, 87, 84, 84, 78, 115, 70, 92, 90, 95, 88, 107, 72, 87, 91, 76, 104, 104, 87, 89, 97, 73, 76, 89, 68, 81, 77, 71, 73, 78, 80, 76, 71, 87, 79, 79, 94, 81, 101, 87, 87, 87, 87, 95, 79, 92, 94, 87, 97, 84, 94, 78, 93, 64, 75, 79, 81, 73, 97, 82, 98, 95, 86, 77, 101, 81, 83, 80, 83, 107, 92, 72, 94, 82, 61, 98, 91, 71, 85, 85, 73, 79, 89, 72, 77, 91, 86, 96, 77, 84, 95, 101, 98, 92, 76, 84, 98, 94, 83, 78, 90, 93, 88, 99, 71, 80, 90, 80, 91, 79, 90, 100, 91, 82, 85, 88, 86, 80, 84, 88, 87, 91, 81, 66, 93, 86, 76, 93, 86, 89, 99, 83, 77, 93, 93, 83, 86, 88, 115, 80, 101, 96, 89, 91, 68, 101, 93, 87, 79, 94, 69, 79, 80, 96, 72, 88, 76, 66, 74, 77, 84, 91, 72, 98, 102, 75, 69, 82, 81, 89, 91, 93, 74, 91, 94] payload_rows_per_dst=[5358, 5300, 5426, 5414, 5446, 5396, 5446, 5440] dup_rows_per_dst=[2798, 2614, 2810, 2838, 2838, 2798, 2800, 2814] combine_rows_per_src=[5358, 5300, 5426, 5414, 5446, 5396, 5446, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43190 payload_rows_remote=37830 dup_rows=22346 dup_rows_remote=19684 dedup_ratio=0.340973 router_tokens_per_expert=[84, 95, 82, 90, 86, 86, 80, 90, 76, 77, 86, 90, 79, 75, 69, 87, 81, 74, 80, 91, 77, 86, 85, 79, 82, 81, 69, 80, 74, 86, 93, 83, 96, 109, 83, 80, 105, 64, 79, 77, 92, 81, 84, 88, 81, 68, 87, 102, 103, 79, 87, 87, 82, 77, 97, 76, 108, 87, 88, 96, 97, 83, 101, 72, 88, 101, 82, 87, 87, 78, 82, 83, 83, 92, 105, 95, 92, 83, 94, 78, 74, 81, 83, 86, 80, 96, 80, 93, 94, 86, 80, 91, 71, 80, 75, 97, 76, 78, 70, 72, 93, 63, 94, 104, 96, 79, 71, 81, 75, 97, 87, 87, 87, 113, 78, 76, 87, 87, 98, 80, 68, 84, 84, 95, 99, 96, 111, 97, 77, 90, 96, 84, 79, 80, 76, 83, 74, 89, 73, 85, 105, 68, 89, 89, 90, 70, 80, 84, 95, 86, 96, 84, 86, 72, 85, 82, 85, 91, 84, 79, 76, 88, 95, 95, 90, 90, 93, 66, 78, 89, 98, 81, 76, 96, 82, 101, 80, 94, 89, 80, 85, 87, 88, 77, 99, 92, 91, 77, 76, 79, 91, 94, 96, 92, 83, 87, 95, 68, 82, 87, 68, 85, 85, 86, 73, 81, 87, 82, 73, 85, 87, 94, 91, 77, 103, 93, 79, 75, 73, 82, 92, 84, 86, 93, 96, 85, 74, 100, 91, 85, 84, 76, 97, 93, 88, 82, 93, 87, 77, 72, 73, 86, 92, 79, 76, 100, 99, 83, 77, 72, 88, 87, 105, 86, 82, 76, 111, 87, 85, 78, 81, 95, 71, 78, 89, 83, 84, 83, 76, 88, 92, 96, 76, 91, 84, 92, 87, 94, 103, 89, 84, 83, 95, 95, 92, 86, 73, 74, 79, 97, 80, 78, 103, 84, 102, 87, 74, 92, 92, 73, 89, 88, 82, 81, 73, 92, 72, 69, 75, 77, 66, 87, 86, 79, 96, 81, 80, 81, 77, 106, 85, 81, 65, 69, 90, 78, 95, 93, 84, 101, 85, 93, 85, 84, 69, 76, 92, 75, 78, 103, 84, 86, 90, 81, 75, 88, 93, 68, 100, 84, 88, 79, 87, 99, 80, 88, 74, 89, 80, 97, 72, 99, 88, 117, 87, 93, 86, 81, 82, 94, 76, 78, 85, 92, 71, 80, 89, 79, 86, 84, 88, 77, 82, 105] payload_rows_per_dst=[5314, 5344, 5340, 5576, 5504, 5412, 5360, 5340] dup_rows_per_dst=[2704, 2826, 2666, 2862, 2848, 2860, 2662, 2918] combine_rows_per_src=[5314, 5344, 5340, 5576, 5504, 5412, 5360, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43470 payload_rows_remote=38138 dup_rows=22066 dup_rows_remote=19304 dedup_ratio=0.336700 router_tokens_per_expert=[75, 66, 95, 84, 84, 74, 82, 91, 103, 108, 86, 74, 85, 86, 80, 85, 94, 91, 84, 66, 89, 78, 80, 75, 79, 74, 87, 98, 96, 92, 87, 94, 69, 89, 73, 86, 80, 80, 98, 78, 82, 80, 79, 88, 91, 94, 104, 83, 96, 86, 76, 78, 97, 82, 99, 96, 100, 88, 101, 80, 81, 90, 98, 79, 88, 80, 86, 94, 81, 95, 80, 85, 67, 79, 81, 75, 81, 74, 85, 72, 88, 70, 81, 71, 108, 85, 83, 78, 91, 95, 91, 80, 92, 94, 101, 87, 78, 83, 71, 80, 90, 77, 89, 83, 94, 81, 86, 82, 99, 84, 71, 91, 99, 86, 78, 92, 81, 90, 97, 98, 92, 89, 83, 83, 97, 94, 85, 101, 86, 95, 79, 95, 83, 78, 95, 76, 83, 83, 86, 91, 90, 92, 73, 65, 82, 79, 70, 77, 91, 68, 87, 74, 75, 85, 91, 88, 74, 79, 89, 92, 90, 94, 77, 85, 84, 82, 82, 89, 82, 83, 75, 83, 102, 90, 82, 79, 88, 86, 78, 91, 78, 78, 83, 84, 92, 80, 88, 79, 74, 86, 79, 103, 93, 88, 94, 90, 99, 73, 80, 73, 89, 99, 73, 77, 84, 83, 88, 96, 76, 76, 82, 97, 82, 79, 85, 84, 83, 73, 82, 89, 79, 86, 93, 89, 85, 75, 83, 95, 74, 82, 94, 73, 104, 78, 76, 84, 73, 91, 99, 87, 86, 92, 85, 78, 75, 84, 76, 90, 87, 93, 81, 83, 85, 101, 97, 70, 75, 87, 73, 75, 79, 91, 85, 98, 82, 103, 82, 74, 77, 84, 90, 75, 95, 79, 79, 81, 83, 88, 75, 79, 95, 79, 85, 109, 89, 74, 74, 90, 86, 70, 86, 80, 76, 99, 83, 85, 83, 90, 92, 93, 85, 80, 80, 82, 81, 83, 87, 98, 73, 78, 84, 96, 94, 85, 78, 102, 88, 92, 83, 86, 87, 105, 84, 86, 81, 105, 89, 91, 95, 90, 96, 99, 70, 101, 80, 97, 80, 89, 76, 81, 85, 89, 94, 92, 85, 75, 80, 85, 74, 83, 92, 82, 93, 79, 73, 82, 84, 74, 76, 93, 87, 96, 95, 93, 97, 86, 85, 94, 93, 87, 85, 93, 79, 99, 90, 85, 86, 82, 93, 97, 67, 92, 95, 66] payload_rows_per_dst=[5470, 5356, 5392, 5460, 5482, 5332, 5518, 5460] dup_rows_per_dst=[2682, 2732, 2708, 2722, 2814, 2762, 2870, 2776] combine_rows_per_src=[5470, 5356, 5392, 5460, 5482, 5332, 5518, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=384 B=48 K=8 R=8 NvS=44960 hidden_size=7168 assignments=32768 payload_rows=43120 payload_rows_remote=37648 dup_rows=22416 dup_rows_remote=19574 dedup_ratio=0.342041 router_tokens_per_expert=[74, 79, 66, 92, 97, 80, 81, 89, 86, 85, 87, 104, 69, 89, 71, 77, 91, 95, 96, 78, 106, 84, 93, 92, 87, 79, 93, 96, 88, 92, 97, 83, 73, 97, 74, 84, 93, 96, 105, 73, 81, 80, 79, 96, 85, 80, 77, 101, 100, 81, 84, 88, 85, 86, 106, 82, 73, 95, 77, 81, 94, 95, 101, 64, 76, 95, 85, 87, 106, 91, 76, 95, 94, 94, 77, 89, 88, 105, 79, 95, 98, 82, 82, 73, 101, 81, 78, 88, 84, 87, 93, 85, 81, 80, 111, 84, 85, 87, 89, 73, 89, 106, 81, 86, 81, 75, 95, 110, 71, 97, 82, 88, 90, 69, 100, 88, 84, 90, 111, 72, 56, 85, 93, 101, 85, 91, 87, 69, 89, 89, 97, 86, 56, 84, 63, 95, 105, 85, 101, 100, 76, 86, 81, 73, 83, 72, 77, 94, 74, 95, 75, 91, 99, 79, 94, 58, 91, 77, 89, 95, 88, 90, 89, 84, 88, 85, 80, 76, 91, 79, 76, 96, 87, 90, 74, 81, 78, 90, 85, 84, 83, 87, 70, 87, 76, 106, 101, 82, 79, 92, 80, 84, 92, 86, 91, 95, 77, 84, 95, 104, 83, 81, 83, 77, 84, 71, 76, 80, 84, 84, 74, 76, 79, 90, 97, 93, 79, 78, 82, 85, 79, 82, 71, 81, 85, 88, 86, 76, 92, 97, 78, 74, 97, 65, 89, 99, 95, 69, 94, 103, 86, 79, 101, 92, 93, 81, 69, 93, 89, 73, 89, 84, 86, 85, 84, 67, 78, 81, 69, 84, 84, 74, 79, 93, 108, 81, 91, 78, 100, 79, 71, 83, 78, 82, 91, 89, 102, 77, 82, 102, 89, 81, 78, 76, 88, 98, 91, 78, 94, 65, 91, 81, 68, 77, 88, 67, 65, 86, 77, 81, 84, 93, 87, 82, 81, 105, 74, 70, 74, 88, 102, 81, 96, 102, 76, 96, 93, 72, 84, 89, 92, 83, 74, 96, 94, 98, 94, 81, 82, 90, 97, 87, 66, 93, 87, 77, 75, 100, 90, 74, 87, 82, 80, 81, 90, 62, 62, 85, 88, 100, 92, 75, 92, 101, 91, 77, 69, 81, 79, 96, 81, 96, 83, 92, 107, 76, 79, 77, 83, 78, 89, 86, 88, 70, 67, 71, 81, 88, 84, 89, 92, 80, 90, 91] payload_rows_per_dst=[5472, 5558, 5466, 5354, 5388, 5444, 5250, 5188] dup_rows_per_dst=[2842, 2880, 3012, 2768, 2732, 2898, 2686, 2598] combine_rows_per_src=[5472, 5558, 5466, 5354, 5388, 5444, 5250, 5188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=96 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541757440 dispatch_meta_bytes=229056 combine_payload_bytes=541757440 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=541986496 ingress_bytes=894078976 link_bytes=894078976 token_bytes=1083743936 weight_movement_bytes=352321536 total_nvlink_bytes=1436065472 weight_to_token_ratio=0.325097 gemm_rows_defined=36096 gemm_rows_charged=45056 gemm_tail_pad_rows=8960 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.198864 gemm_active_groups=50 gemm_peak_group_rows=896 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.509703588e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540151808 dispatch_meta_bytes=229512 combine_payload_bytes=540151808 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=540381320 ingress_bytes=716312576 link_bytes=716312576 token_bytes=1080533128 weight_movement_bytes=176160768 total_nvlink_bytes=1256693896 weight_to_token_ratio=0.163031 gemm_rows_defined=36736 gemm_rows_charged=45056 gemm_tail_pad_rows=8320 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.184659 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.016577133e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=543592448 dispatch_meta_bytes=229824 combine_payload_bytes=543592448 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=543822272 ingress_bytes=719753216 link_bytes=719753216 token_bytes=1087414720 weight_movement_bytes=176160768 total_nvlink_bytes=1263575488 weight_to_token_ratio=0.162000 gemm_rows_defined=36736 gemm_rows_charged=45056 gemm_tail_pad_rows=8320 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.184659 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=540696576 dispatch_meta_bytes=228992 combine_payload_bytes=540696576 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=540925568 ingress_bytes=893018112 link_bytes=893018112 token_bytes=1081622144 weight_movement_bytes=352321536 total_nvlink_bytes=1433943680 weight_to_token_ratio=0.325734 gemm_rows_defined=36224 gemm_rows_charged=45056 gemm_tail_pad_rows=8832 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.196023 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.500792995e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=541700096 dispatch_meta_bytes=229128 combine_payload_bytes=541700096 prefetch_bytes=0 reduce_bytes=704643072 egress_bytes=541929224 ingress_bytes=1246343168 link_bytes=1246343168 token_bytes=1083629320 weight_movement_bytes=704643072 total_nvlink_bytes=1788272392 weight_to_token_ratio=0.650262 gemm_rows_defined=36480 gemm_rows_charged=45056 gemm_tail_pad_rows=8576 gemm_inner_pad_upper_bound=6223 gemm_tail_pad_fraction=0.190341 gemm_active_groups=49 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.046850224e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=542330880 dispatch_meta_bytes=230056 combine_payload_bytes=542330880 prefetch_bytes=176160768 reduce_bytes=176160768 egress_bytes=542560936 ingress_bytes=894652416 link_bytes=894652416 token_bytes=1084891816 weight_movement_bytes=352321536 total_nvlink_bytes=1437213352 weight_to_token_ratio=0.324753 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514520125e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=94 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.020833 weight_bytes_per_expert=88080384 grad_bytes_per_expert=176160768 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083743936 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=541986496 ingress_bytes=894078976 link_bytes=894078976 total_nvlink_bytes=1436065472 weight_to_token_ratio=0.325097 prefetch_share_of_ingress=0.197030 dedup_ratio=0.340363 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.198864 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.509703588e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=546746368 dispatch_meta_bytes=229768 combine_payload_bytes=546746368 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=546976136 ingress_bytes=722907136 link_bytes=722907136 token_bytes=1093722504 weight_movement_bytes=176160768 total_nvlink_bytes=1269883272 weight_to_token_ratio=0.161065 gemm_rows_defined=36352 gemm_rows_charged=45056 gemm_tail_pad_rows=8704 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.193182 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.071967308e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1080533128 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=540381320 ingress_bytes=716312576 link_bytes=716312576 total_nvlink_bytes=1256693896 weight_to_token_ratio=0.163031 prefetch_share_of_ingress=0.245927 dedup_ratio=0.343689 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.184659 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.016577133e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1087414720 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=543822272 ingress_bytes=719753216 link_bytes=719753216 total_nvlink_bytes=1263575488 weight_to_token_ratio=0.162000 prefetch_share_of_ingress=0.244752 dedup_ratio=0.340271 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.184659 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.045476355e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1081622144 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=540925568 ingress_bytes=893018112 link_bytes=893018112 total_nvlink_bytes=1433943680 weight_to_token_ratio=0.325734 prefetch_share_of_ingress=0.197264 dedup_ratio=0.340454 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.196023 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.500792995e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=539721728 dispatch_meta_bytes=228888 combine_payload_bytes=539721728 prefetch_bytes=176160768 reduce_bytes=0 egress_bytes=539950616 ingress_bytes=715882496 link_bytes=715882496 token_bytes=1079672344 weight_movement_bytes=176160768 total_nvlink_bytes=1255833112 weight_to_token_ratio=0.163161 gemm_rows_defined=36608 gemm_rows_charged=45056 gemm_tail_pad_rows=8448 gemm_inner_pad_upper_bound=6350 gemm_tail_pad_fraction=0.187500 gemm_active_groups=50 gemm_peak_group_rows=768 gemm_flops_fwd=3968549781504 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.012964730e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1083629320 prefetch_bytes=0 reduce_bytes=704643072 weight_movement_bytes=704643072 egress_bytes=541929224 ingress_bytes=1246343168 link_bytes=1246343168 total_nvlink_bytes=1788272392 weight_to_token_ratio=0.650262 prefetch_share_of_ingress=0.000000 dedup_ratio=0.340424 prefetch_slots=0 unused_prefetch_slots=96 slot_utilization=0.000000 reduce_slots=4 remote_demand=0 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.190341 gemm_inner_pad_upper_bound=6223 gemm_flops_step=11905649344512 bytes_per_gemm_flop=1.046850224e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1084891816 prefetch_bytes=176160768 reduce_bytes=176160768 weight_movement_bytes=352321536 egress_bytes=542560936 ingress_bytes=894652416 link_bytes=894652416 total_nvlink_bytes=1437213352 weight_to_token_ratio=0.324753 prefetch_share_of_ingress=0.196904 dedup_ratio=0.340973 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=1 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=7.514520125e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1093722504 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=546976136 ingress_bytes=722907136 link_bytes=722907136 total_nvlink_bytes=1269883272 weight_to_token_ratio=0.161065 prefetch_share_of_ingress=0.243684 dedup_ratio=0.336700 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.193182 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.071967308e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=1 plans=1 dispatch_calls=2 combine_calls=2 prefetch_calls=2 reduce_calls=1 token_bytes=1079672344 prefetch_bytes=176160768 reduce_bytes=0 weight_movement_bytes=176160768 egress_bytes=539950616 ingress_bytes=715882496 link_bytes=715882496 total_nvlink_bytes=1255833112 weight_to_token_ratio=0.163161 prefetch_share_of_ingress=0.246075 dedup_ratio=0.342041 prefetch_slots=2 unused_prefetch_slots=94 slot_utilization=0.020833 reduce_slots=0 remote_demand=1 gemm_rows_charged=45056 gemm_tail_pad_fraction=0.187500 gemm_inner_pad_upper_bound=6350 gemm_flops_step=11905649344512 bytes_per_gemm_flop=6.012964730e-05 prefetch_bytes_per_gemm_flop=1.479640152e-05 anomalies=none + [2026-09-21 07:42:56] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 120.4 | throughput per GPU (TFLOP/s/GPU): 293.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.043282E+01 | loss scale: 1.0 | grad norm: 8.198 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 07:42:56 +[rank0]:[W921 07:42:57.470703458 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W921 07:42:58.420295647 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W921 07:42:58.810174357 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W921 07:42:58.812542725 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W921 07:42:59.172977081 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W921 07:42:59.173102129 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W921 07:42:59.173882357 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W921 07:42:59.208048038 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 07:43:04.929000 498333 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 07:43:04.930000 498333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 498525 closing signal SIGTERM +W0921 07:43:04.930000 498333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 498528 closing signal SIGTERM +W0921 07:43:04.931000 498333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 498531 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 498333 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 9edea848dbf0b9ad41f4114bcb6d0b0049c47431..f63467dd2913fd726e98d16b6993b1548c6e7084 100644 --- a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,2 +1,3 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir KIMI_K2_L2 moonep natural none OK 86157 1 0 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 +KIMI_K2_L2 moonep forced none OK 86157 1 0 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 39d3ade27d5b209764c83daa5db58c8c54a0b215..3fc834b5105c6f6ee8b5ad0cd0613f63690e6377 100644 --- a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1 +1,2 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +KIMI_K2_L2 moonep SELECTED none 0 86157 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d02706708a61069ab8c8e780c362cb9f152c4784 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,228 @@ +[ + { + "arm": "natural", + "aux_loss_coeff": "0", + "backend": "moonep", + "branch": "moonep_backend_async_grad_reduce_token_routing_analysis", + "capacity": "none", + "capacity_mode": "none", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 10, + "commit": "a6a1a221", + "config": "KIMI_K2_L2", + "cuda_graph": "0", + "dense_first_layers": "1", + "experts_per_rank": "48", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 10, + "hottest_expert_token_fraction_max": 0.2982177734375, + "hottest_expert_token_fraction_mean": 0.170477294921875, + "hottest_expert_token_fraction_min": 0.110565185546875, + "hottest_expert_token_fraction_n": 10, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "1", + "moonep_bytes_global_snapshots": 10, + "moonep_bytes_incomplete_snapshots": 0, + "moonep_bytes_malformed_records": 0, + "moonep_global_combine_payload_bytes_max": 4395991040.0, + "moonep_global_combine_payload_bytes_mean": 4330650419.2, + "moonep_global_combine_payload_bytes_min": 4301602816.0, + "moonep_global_combine_payload_bytes_n": 10, + "moonep_global_dispatch_payload_bytes_max": 4395991040.0, + "moonep_global_dispatch_payload_bytes_mean": 4330650419.2, + "moonep_global_dispatch_payload_bytes_min": 4301602816.0, + "moonep_global_dispatch_payload_bytes_n": 10, + "moonep_global_prefetch_bytes_max": 1937768448.0, + "moonep_global_prefetch_bytes_mean": 1603062988.8, + "moonep_global_prefetch_bytes_min": 1233125376.0, + "moonep_global_prefetch_bytes_n": 10, + "moonep_global_reduce_bytes_max": 1937768448.0, + "moonep_global_reduce_bytes_mean": 1603062988.8, + "moonep_global_reduce_bytes_min": 1233125376.0, + "moonep_global_reduce_bytes_n": 10, + "moonep_global_token_bytes_max": 8793816424.0, + "moonep_global_token_bytes_mean": 8663126381.6, + "moonep_global_token_bytes_min": 8605026376.0, + "moonep_global_token_bytes_n": 10, + "moonep_global_total_nvlink_bytes_max": 12480563272.0, + "moonep_global_total_nvlink_bytes_mean": 11869252359.2, + "moonep_global_total_nvlink_bytes_min": 11211418776.0, + "moonep_global_total_nvlink_bytes_n": 10, + "moonep_global_weight_movement_bytes_max": 3875536896.0, + "moonep_global_weight_movement_bytes_mean": 3206125977.6, + "moonep_global_weight_movement_bytes_min": 2466250752.0, + "moonep_global_weight_movement_bytes_n": 10, + "moonep_global_weight_to_token_ratio_max": 0.4503805946265469, + "moonep_global_weight_to_token_ratio_mean": 0.37043439150680124, + "moonep_global_weight_to_token_ratio_min": 0.280452835616301, + "moonep_global_weight_to_token_ratio_n": 10, + "moonep_remote_demand_max": 2.0, + "moonep_remote_demand_mean": 1.1375, + "moonep_remote_demand_min": 0.0, + "moonep_remote_demand_n": 80, + "moonep_slot_utilization_max": 0.041667, + "moonep_slot_utilization_mean": 0.0236978, + "moonep_slot_utilization_min": 0.0, + "moonep_slot_utilization_n": 80, + "moonep_unused_prefetch_slots_max": 96.0, + "moonep_unused_prefetch_slots_mean": 93.725, + "moonep_unused_prefetch_slots_min": 92.0, + "moonep_unused_prefetch_slots_n": 80, + "num_experts": "384", + "num_layers": "2", + "post_physical_rank_cv_max": 0.022939437595188024, + "post_physical_rank_cv_mean": 0.016598105480502157, + "post_physical_rank_cv_min": 0.0017300051649072432, + "post_physical_rank_cv_n": 10, + "post_physical_rank_imax_max": 1.0368468027524256, + "post_physical_rank_imax_mean": 1.0260113076815665, + "post_physical_rank_imax_min": 1.003556609365738, + "post_physical_rank_imax_n": 10, + "pre_home_rank_cv_max": 0.23147253286871253, + "pre_home_rank_cv_mean": 0.1336617698353712, + "pre_home_rank_cv_min": 0.08928583511808427, + "pre_home_rank_cv_n": 10, + "pre_home_rank_imax_max": 1.389129638671875, + "pre_home_rank_imax_mean": 1.230499267578125, + "pre_home_rank_imax_min": 1.129058837890625, + "pre_home_rank_imax_n": 10, + "raw_expert_cv_max": 1.7760509001770588, + "raw_expert_cv_mean": 1.0733812106366694, + "raw_expert_cv_min": 0.7388867791341617, + "raw_expert_cv_n": 10, + "raw_expert_imax_ceiling": 48.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 14.314453125, + "raw_expert_imax_mean": 8.18291015625, + "raw_expert_imax_min": 5.30712890625, + "raw_expert_imax_n": 10, + "records": { + "moonep_bytes": 80, + "moonep_routing": 80, + "moonep_summary": 80, + "moonep_weights": 80 + }, + "router_balancing": "seq_aux_loss", + "run_dir": "KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8" + }, + { + "arm": "forced", + "aux_loss_coeff": "0", + "backend": "moonep", + "branch": "moonep_backend_async_grad_reduce_token_routing_analysis", + "capacity": "none", + "capacity_mode": "none", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 10, + "commit": "a6a1a221", + "config": "KIMI_K2_L2", + "cuda_graph": "0", + "dense_first_layers": "1", + "experts_per_rank": "48", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 10, + "hottest_expert_token_fraction_max": 0.0240478515625, + "hottest_expert_token_fraction_mean": 0.0235992431640625, + "hottest_expert_token_fraction_min": 0.02313232421875, + "hottest_expert_token_fraction_n": 10, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "1", + "moonep_bytes_global_snapshots": 10, + "moonep_bytes_incomplete_snapshots": 0, + "moonep_bytes_malformed_records": 0, + "moonep_global_combine_payload_bytes_max": 4344438784.0, + "moonep_global_combine_payload_bytes_mean": 4338601164.8, + "moonep_global_combine_payload_bytes_min": 4333916160.0, + "moonep_global_combine_payload_bytes_n": 10, + "moonep_global_dispatch_payload_bytes_max": 4344438784.0, + "moonep_global_dispatch_payload_bytes_mean": 4338601164.8, + "moonep_global_dispatch_payload_bytes_min": 4333916160.0, + "moonep_global_dispatch_payload_bytes_n": 10, + "moonep_global_prefetch_bytes_max": 1233125376.0, + "moonep_global_prefetch_bytes_mean": 1233125376.0, + "moonep_global_prefetch_bytes_min": 1233125376.0, + "moonep_global_prefetch_bytes_n": 10, + "moonep_global_reduce_bytes_max": 1233125376.0, + "moonep_global_reduce_bytes_mean": 1233125376.0, + "moonep_global_reduce_bytes_min": 1233125376.0, + "moonep_global_reduce_bytes_n": 10, + "moonep_global_token_bytes_max": 8690713000.0, + "moonep_global_token_bytes_mean": 8679036183.2, + "moonep_global_token_bytes_min": 8669663568.0, + "moonep_global_token_bytes_n": 10, + "moonep_global_total_nvlink_bytes_max": 11156963752.0, + "moonep_global_total_nvlink_bytes_mean": 11145286935.2, + "moonep_global_total_nvlink_bytes_min": 11135914320.0, + "moonep_global_total_nvlink_bytes_n": 10, + "moonep_global_weight_movement_bytes_max": 2466250752.0, + "moonep_global_weight_movement_bytes_mean": 2466250752.0, + "moonep_global_weight_movement_bytes_min": 2466250752.0, + "moonep_global_weight_movement_bytes_n": 10, + "moonep_global_weight_to_token_ratio_max": 0.2844690261226524, + "moonep_global_weight_to_token_ratio_mean": 0.28416198294879025, + "moonep_global_weight_to_token_ratio_min": 0.2837800249530735, + "moonep_global_weight_to_token_ratio_n": 10, + "moonep_remote_demand_max": 1.0, + "moonep_remote_demand_mean": 0.875, + "moonep_remote_demand_min": 0.0, + "moonep_remote_demand_n": 80, + "moonep_slot_utilization_max": 0.020833, + "moonep_slot_utilization_mean": 0.018228875, + "moonep_slot_utilization_min": 0.0, + "moonep_slot_utilization_n": 80, + "moonep_unused_prefetch_slots_max": 96.0, + "moonep_unused_prefetch_slots_mean": 94.25, + "moonep_unused_prefetch_slots_min": 94.0, + "moonep_unused_prefetch_slots_n": 80, + "num_experts": "384", + "num_layers": "2", + "post_physical_rank_cv_max": 0.0031320054168418494, + "post_physical_rank_cv_mean": 0.0022532777674585456, + "post_physical_rank_cv_min": 0.0014149963886086814, + "post_physical_rank_cv_n": 10, + "post_physical_rank_imax_max": 1.004496538012507, + "post_physical_rank_imax_mean": 1.0033084889047725, + "post_physical_rank_imax_min": 1.0019296319833613, + "post_physical_rank_imax_n": 10, + "pre_home_rank_cv_max": 0.009327227347999425, + "pre_home_rank_cv_mean": 0.007192100098074482, + "pre_home_rank_cv_min": 0.003541485721042321, + "pre_home_rank_cv_n": 10, + "pre_home_rank_imax_max": 1.02001953125, + "pre_home_rank_imax_mean": 1.01103515625, + "pre_home_rank_imax_min": 1.005340576171875, + "pre_home_rank_imax_n": 10, + "raw_expert_cv_max": 0.04836014600641425, + "raw_expert_cv_mean": 0.045649240379339776, + "raw_expert_cv_min": 0.04141931327746496, + "raw_expert_cv_n": 10, + "raw_expert_imax_ceiling": 48.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.154296875, + "raw_expert_imax_mean": 1.132763671875, + "raw_expert_imax_min": 1.1103515625, + "raw_expert_imax_n": 10, + "records": { + "moonep_bytes": 80, + "moonep_routing": 80, + "moonep_summary": 80, + "moonep_weights": 80 + }, + "router_balancing": "seq_aux_loss", + "run_dir": "KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8" + } +] \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e0c3e8319cd6d35f68a5c8372e75822acb7fe272 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,3 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n +KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural KIMI_K2_L2 moonep 384 8 48 2 1 1 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 0 0 0.0 0.0 0.0 10 10 0.2982177734375 0.170477294921875 0.110565185546875 10 0 0 10 0 0 4395991040.0 4330650419.2 4301602816.0 10 4395991040.0 4330650419.2 4301602816.0 10 1937768448.0 1603062988.8 1233125376.0 10 1937768448.0 1603062988.8 1233125376.0 10 8793816424.0 8663126381.6 8605026376.0 10 12480563272.0 11869252359.2 11211418776.0 10 3875536896.0 3206125977.6 2466250752.0 10 0.4503805946265469 0.37043439150680124 0.280452835616301 10 2.0 1.1375 0.0 80 0.041667 0.0236978 0.0 80 96.0 93.725 92.0 80 0.022939437595188024 0.016598105480502157 0.0017300051649072432 10 1.0368468027524256 1.0260113076815665 1.003556609365738 10 0.23147253286871253 0.1336617698353712 0.08928583511808427 10 1.389129638671875 1.230499267578125 1.129058837890625 10 1.7760509001770588 1.0733812106366694 0.7388867791341617 10 48.0 0 14.314453125 8.18291015625 5.30712890625 10 +KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced KIMI_K2_L2 moonep 384 8 48 2 1 1 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 0 0 0.0 0.0 0.0 10 10 0.0240478515625 0.0235992431640625 0.02313232421875 10 0 0 10 0 0 4344438784.0 4338601164.8 4333916160.0 10 4344438784.0 4338601164.8 4333916160.0 10 1233125376.0 1233125376.0 1233125376.0 10 1233125376.0 1233125376.0 1233125376.0 10 8690713000.0 8679036183.2 8669663568.0 10 11156963752.0 11145286935.2 11135914320.0 10 2466250752.0 2466250752.0 2466250752.0 10 0.2844690261226524 0.28416198294879025 0.2837800249530735 10 1.0 0.875 0.0 80 0.020833 0.018228875 0.0 80 96.0 94.25 94.0 80 0.0031320054168418494 0.0022532777674585456 0.0014149963886086814 10 1.004496538012507 1.0033084889047725 1.0019296319833613 10 0.009327227347999425 0.007192100098074482 0.003541485721042321 10 1.02001953125 1.01103515625 1.005340576171875 10 0.04836014600641425 0.045649240379339776 0.04141931327746496 10 48.0 0 1.154296875 1.132763671875 1.1103515625 10 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/results.tsv index dfbec4f0ff9b032878c5e0cc6bf8387b2fce59f9..5712acc26db2e230305b51bf0e53b16422034226 100644 --- a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,2 +1,3 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir KIMI_K2_L2 moonep matched 48 4096 8 none none 1 OK 121.60 120.60 86157 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 +KIMI_K2_L2 moonep matched 48 4096 8 none none+forcebal 1 OK 120.39 119.90 86157 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1